diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/conftest.py b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/conftest.py index f58ee7224089d..b2e2c416bc3c9 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/conftest.py @@ -6,12 +6,15 @@ import json import pytest +from source_facebook_marketing.config_migrations import MigrateAccountIdToArray @pytest.fixture(scope="session", name="config") def config_fixture(): with open("secrets/config.json", "r") as config_file: - return json.load(config_file) + config = json.load(config_file) + migrated_config = MigrateAccountIdToArray.transform(config) + return migrated_config @pytest.fixture(scope="session", name="config_with_wrong_token") @@ -21,7 +24,7 @@ def config_with_wrong_token_fixture(config): @pytest.fixture(scope="session", name="config_with_wrong_account") def config_with_wrong_account_fixture(config): - return {**config, "account_id": "WRONG_ACCOUNT"} + return {**config, "account_ids": ["WRONG_ACCOUNT"]} @pytest.fixture(scope="session", name="config_with_include_deleted") diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl index 6bed2dde636d5..92fc12d2f7103 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl @@ -4,7 +4,7 @@ {"stream":"campaigns","data":{"id":"23846542053890398","account_id":"212551616838260","budget_rebalance_flag":false,"budget_remaining":0.0,"buying_type":"AUCTION","created_time":"2021-01-18T21:36:42-0800","configured_status":"PAUSED","effective_status":"PAUSED","name":"Fake Campaign 0","objective":"MESSAGES","smart_promotion_type":"GUIDED_CREATION","source_campaign_id":0.0,"special_ad_category":"NONE","start_time":"1969-12-31T15:59:59-0800","status":"PAUSED","updated_time":"2021-02-18T01:00:02-0800"},"emitted_at":1694795155769} {"stream": "custom_audiences", "data": {"id": "23853683587660398", "account_id": "212551616838260", "approximate_count_lower_bound": 4700, "approximate_count_upper_bound": 5500, "customer_file_source": "PARTNER_PROVIDED_ONLY", "data_source": {"type": "UNKNOWN", "sub_type": "ANYTHING", "creation_params": "[]"}, "delivery_status": {"code": 200, "description": "This audience is ready for use."}, "description": "Custom Audience-Web Traffic [ALL] - _copy", "is_value_based": false, "name": "Web Traffic [ALL] - _copy", "operation_status": {"code": 200, "description": "Normal"}, "permission_for_actions": {"can_edit": true, "can_see_insight": "True", "can_share": "True", "subtype_supports_lookalike": "True", "supports_recipient_lookalike": "False"}, "retention_days": 0, "subtype": "CUSTOM", "time_content_updated": 1679433484, "time_created": 1679433479, "time_updated": 1679433484}, "emitted_at": 1698925454024} {"stream":"ad_creatives","data":{"id":"23844568440620398","account_id":"212551616838260","actor_id":"112704783733939","asset_feed_spec":{"images":[{"adlabels":[{"name":"placement_asset_fb19ee1baacc68_1586830094862","id":"23844521781280398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"191x100":[[0,411],[589,719]]}},{"adlabels":[{"name":"placement_asset_f1f518506ae7e68_1586830094842","id":"23844521781340398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"100x100":[[12,282],[574,844]]}},{"adlabels":[{"name":"placement_asset_f311b79c14a30c_1586830094845","id":"23844521781330398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"90x160":[[14,72],[562,1046]]}},{"adlabels":[{"name":"placement_asset_f2c2fe4f20af66c_1586830157386","id":"23844521783780398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"90x160":[[0,0],[589,1047]]}}],"bodies":[{"adlabels":[{"name":"placement_asset_f2d65f15340e594_1586830094852","id":"23844521781260398"},{"name":"placement_asset_f1f97c3e3a63d74_1586830094858","id":"23844521781300398"},{"name":"placement_asset_f14cee2ab5d786_1586830094863","id":"23844521781370398"},{"name":"placement_asset_f14877915fb5acc_1586830157387","id":"23844521783760398"}],"text":""}],"call_to_action_types":["LEARN_MORE"],"descriptions":[{"text":"Unmatched attribution, ad performances, and lead conversion, by unlocking your ad-blocked traffic across all your tools."}],"link_urls":[{"adlabels":[{"name":"placement_asset_f309294689f2c6c_1586830094864","id":"23844521781290398"},{"name":"placement_asset_f136a02466f2bc_1586830094856","id":"23844521781310398"},{"name":"placement_asset_fa79b032b68274_1586830094860","id":"23844521781320398"},{"name":"placement_asset_f28a128696c7428_1586830157387","id":"23844521783790398"}],"website_url":"http://dataline.io/","display_url":""}],"titles":[{"adlabels":[{"name":"placement_asset_f1013e29f89c38_1586830094864","id":"23844521781350398"},{"name":"placement_asset_fcb53b78a11574_1586830094859","id":"23844521781360398"},{"name":"placement_asset_f1a3b3d525f4998_1586830094854","id":"23844521781380398"},{"name":"placement_asset_f890656071c9ac_1586830157387","id":"23844521783770398"}],"text":"Unblock all your adblocked traffic"}],"ad_formats":["AUTOMATIC_FORMAT"],"asset_customization_rules":[{"customization_spec":{"age_max":65,"age_min":13,"publisher_platforms":["instagram","audience_network","messenger"],"instagram_positions":["story"],"messenger_positions":["story"],"audience_network_positions":["classic"]},"image_label":{"name":"placement_asset_f311b79c14a30c_1586830094845","id":"23844521781330398"},"body_label":{"name":"placement_asset_f1f97c3e3a63d74_1586830094858","id":"23844521781300398"},"link_url_label":{"name":"placement_asset_fa79b032b68274_1586830094860","id":"23844521781320398"},"title_label":{"name":"placement_asset_fcb53b78a11574_1586830094859","id":"23844521781360398"},"priority":1},{"customization_spec":{"age_max":65,"age_min":13,"publisher_platforms":["facebook"],"facebook_positions":["right_hand_column","instant_article","search"]},"image_label":{"name":"placement_asset_fb19ee1baacc68_1586830094862","id":"23844521781280398"},"body_label":{"name":"placement_asset_f14cee2ab5d786_1586830094863","id":"23844521781370398"},"link_url_label":{"name":"placement_asset_f309294689f2c6c_1586830094864","id":"23844521781290398"},"title_label":{"name":"placement_asset_f1013e29f89c38_1586830094864","id":"23844521781350398"},"priority":2},{"customization_spec":{"age_max":65,"age_min":13,"publisher_platforms":["facebook"],"facebook_positions":["story"]},"image_label":{"name":"placement_asset_f2c2fe4f20af66c_1586830157386","id":"23844521783780398"},"body_label":{"name":"placement_asset_f14877915fb5acc_1586830157387","id":"23844521783760398"},"link_url_label":{"name":"placement_asset_f28a128696c7428_1586830157387","id":"23844521783790398"},"title_label":{"name":"placement_asset_f890656071c9ac_1586830157387","id":"23844521783770398"},"priority":3},{"customization_spec":{"age_max":65,"age_min":13},"image_label":{"name":"placement_asset_f1f518506ae7e68_1586830094842","id":"23844521781340398"},"body_label":{"name":"placement_asset_f2d65f15340e594_1586830094852","id":"23844521781260398"},"link_url_label":{"name":"placement_asset_f136a02466f2bc_1586830094856","id":"23844521781310398"},"title_label":{"name":"placement_asset_f1a3b3d525f4998_1586830094854","id":"23844521781380398"},"priority":4}],"optimization_type":"PLACEMENT","reasons_to_shop":false,"shops_bundle":false,"additional_data":{"multi_share_end_card":false,"is_click_to_message":false}},"effective_object_story_id":"112704783733939_117519556585795","name":"{{product.name}} 2020-04-21-49cbe5bd90ed9861ea68bb38f7d6fc7c","instagram_actor_id":"3437258706290825","object_story_spec":{"page_id":"112704783733939","instagram_actor_id":"3437258706290825"},"object_type":"SHARE","status":"ACTIVE","thumbnail_url":"https://scontent-dus1-1.xx.fbcdn.net/v/t45.1600-4/93287504_23844521781140398_125048020067680256_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=a3999f&_nc_ohc=-TT4Z0FkPeYAX97qejq&_nc_ht=scontent-dus1-1.xx&edm=AAT1rw8EAAAA&stp=c0.5000x0.5000f_dst-emg0_p64x64_q75&ur=58080a&oh=00_AfBjMrayWFyOLmIgVt8Owtv2fBSJVyCmtNuPLpCQyggdpg&oe=64E18154"},"emitted_at":1692180825964} -{"stream":"activities","data":{"actor_id":"122043039268043192","actor_name":"Payments RTU Processor","application_id":"0","date_time_in_timezone":"03/13/2023 at 6:30 AM","event_time":"2023-03-13T13:30:47+0000","event_type":"ad_account_billing_charge","extra_data":"{\"currency\":\"USD\",\"new_value\":1188,\"transaction_id\":\"5885578541558696-11785530\",\"action\":67,\"type\":\"payment_amount\"}","object_id":"212551616838260","object_name":"Airbyte","object_type":"ACCOUNT","translated_event_type":"Account billed"},"emitted_at":1696931251153} +{"stream":"activities","data":{"account_id":"212551616838260","actor_id":"122043039268043192","actor_name":"Payments RTU Processor","application_id":"0","date_time_in_timezone":"03/13/2023 at 6:30 AM","event_time":"2023-03-13T13:30:47+0000","event_type":"ad_account_billing_charge","extra_data":"{\"currency\":\"USD\",\"new_value\":1188,\"transaction_id\":\"5885578541558696-11785530\",\"action\":67,\"type\":\"payment_amount\"}","object_id":"212551616838260","object_name":"Airbyte","object_type":"ACCOUNT","translated_event_type":"Account billed"},"emitted_at":1696931251153} {"stream":"custom_conversions","data":{"id":"694166388077667","account_id":"212551616838260","creation_time":"2020-04-22T01:36:00+0000","custom_event_type":"CONTACT","data_sources":[{"id":"2667253716886462","source_type":"PIXEL","name":"Dataline's Pixel"}],"default_conversion_value":0,"event_source_type":"pixel","is_archived":true,"is_unavailable":false,"name":"SubscribedButtonClick","retention_days":0,"rule":"{\"and\":[{\"event\":{\"eq\":\"PageView\"}},{\"or\":[{\"URL\":{\"i_contains\":\"SubscribedButtonClick\"}}]}]}"},"emitted_at":1692180839174} {"stream":"images","data":{"id":"212551616838260:c1e94a8768a405f0f212d71fe8336647","account_id":"212551616838260","name":"Audience_1_Ad_3_1200x1200_blue_CTA_arrow.png_105","creatives":["23853630775340398","23853630871360398","23853666124200398"],"original_height":1200,"original_width":1200,"permalink_url":"https://www.facebook.com/ads/image/?d=AQIDNjjLb7VzVJ26jXb_HpudCEUJqbV_lLF2JVsdruDcBxnXQEKfzzd21VVJnkm0B-JLosUXNNg1BH78y7FxnK3AH-0D_lnk7kn39_bIcOMK7Z9HYyFInfsVY__adup3A5zGTIcHC9Y98Je5qK-yD8F6","status":"ACTIVE","url":"https://scontent-dus1-1.xx.fbcdn.net/v/t45.1600-4/335907140_23853620220420398_4375584095210967511_n.png?_nc_cat=104&ccb=1-7&_nc_sid=2aac32&_nc_ohc=xdjrPpbRGNAAX8Dck01&_nc_ht=scontent-dus1-1.xx&edm=AJcBmwoEAAAA&oh=00_AfDCqQ6viqrgLcfbO3O5-n030Usq7Zyt2c1TmsatqnYf7Q&oe=64E2779A","created_time":"2023-03-16T13:13:17-0700","hash":"c1e94a8768a405f0f212d71fe8336647","url_128":"https://scontent-dus1-1.xx.fbcdn.net/v/t45.1600-4/335907140_23853620220420398_4375584095210967511_n.png?stp=dst-png_s128x128&_nc_cat=104&ccb=1-7&_nc_sid=2aac32&_nc_ohc=xdjrPpbRGNAAX8Dck01&_nc_ht=scontent-dus1-1.xx&edm=AJcBmwoEAAAA&oh=00_AfAY50CMpox2s4w_f18IVx7sZuXlg4quF6YNIJJ8D4PZew&oe=64E2779A","is_associated_creatives_in_adgroups":true,"updated_time":"2023-03-17T08:09:56-0700","height":1200,"width":1200},"emitted_at":1692180839582} {"stream":"ads_insights","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","actions":[{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"page_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"post_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"link_click","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0}],"ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":3,"conversion_rate_ranking":"UNKNOWN","cost_per_estimated_ad_recallers":0.007,"cost_per_inline_link_click":0.396667,"cost_per_inline_post_engagement":0.396667,"cost_per_unique_click":0.396667,"cost_per_unique_inline_link_click":0.396667,"cpc":0.396667,"cpm":0.902199,"cpp":0.948207,"created_time":"2021-02-09","ctr":0.227445,"date_start":"2021-02-15","date_stop":"2021-02-15","engagement_rate_ranking":"UNKNOWN","estimated_ad_recall_rate":13.545817,"estimated_ad_recallers":170.0,"frequency":1.050996,"impressions":1319,"inline_link_click_ctr":0.227445,"inline_link_clicks":3,"inline_post_engagement":3,"instant_experience_clicks_to_open":1.0,"instant_experience_clicks_to_start":1.0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","outbound_clicks":[{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"outbound_click","value":3.0}],"quality_ranking":"UNKNOWN","reach":1255,"social_spend":0.0,"spend":1.19,"unique_actions":[{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"page_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"post_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"link_click","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0}],"unique_clicks":3,"unique_ctr":0.239044,"unique_inline_link_click_ctr":0.239044,"unique_inline_link_clicks":3,"unique_link_clicks_ctr":0.239044,"unique_outbound_clicks":[{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"outbound_click","value":3.0}],"updated_time":"2021-08-27","video_play_curve_actions":[{"action_type":"video_view"}],"website_ctr":[{"action_type":"link_click","value":0.227445}],"wish_bid":0.0},"emitted_at":1682686057366} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index 1719883cf5cab..1657aaeda2d0d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -5,14 +5,19 @@ "title": "Source Facebook Marketing", "type": "object", "properties": { - "account_id": { - "title": "Ad Account ID", - "description": "The Facebook Ad account ID to use when pulling data from the Facebook Marketing API. The Ad account ID number is in the account dropdown menu or in your browser's address bar of your Meta Ads Manager. See the docs for more information.", + "account_ids": { + "title": "Ad Account ID(s)", + "description": "The Facebook Ad account ID(s) to pull data from. The Ad account ID number is in the account dropdown menu or in your browser's address bar of your Meta Ads Manager. See the docs for more information.", "order": 0, - "pattern": "^[0-9]+$", - "pattern_descriptor": "1234567890", + "pattern_descriptor": "The Ad Account ID must be a number.", "examples": ["111111111111111"], - "type": "string" + "type": "array", + "minItems": 1, + "items": { + "pattern": "^[0-9]+$", + "type": "string" + }, + "uniqueItems": true }, "access_token": { "title": "Access Token", @@ -388,7 +393,7 @@ "type": "string" } }, - "required": ["account_id", "access_token"] + "required": ["account_ids", "access_token"] }, "supportsIncremental": true, "supported_destination_sync_modes": ["append"], diff --git a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml index faf2e1fa31bcb..0bac6b20fc1de 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c - dockerImageTag: 1.2.3 + dockerImageTag: 1.3.0 dockerRepository: airbyte/source-facebook-marketing documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing githubIssueLabel: source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py index e3a6c610e117a..61a171b9659d2 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py @@ -6,10 +6,10 @@ import logging from dataclasses import dataclass from time import sleep +from typing import List import backoff import pendulum -from cached_property import cached_property from facebook_business import FacebookAdsApi from facebook_business.adobjects.adaccount import AdAccount from facebook_business.api import FacebookResponse @@ -173,8 +173,8 @@ def call( class API: """Simple wrapper around Facebook API""" - def __init__(self, account_id: str, access_token: str, page_size: int = 100): - self._account_id = account_id + def __init__(self, access_token: str, page_size: int = 100): + self._accounts = {} # design flaw in MyFacebookAdsApi requires such strange set of new default api instance self.api = MyFacebookAdsApi.init(access_token=access_token, crash_log=False) # adding the default page size from config to the api base class @@ -183,10 +183,12 @@ def __init__(self, account_id: str, access_token: str, page_size: int = 100): # set the default API client to Facebook lib. FacebookAdsApi.set_default_api(self.api) - @cached_property - def account(self) -> AdAccount: - """Find current account""" - return self._find_account(self._account_id) + def get_account(self, account_id: str) -> AdAccount: + """Get AdAccount object by id""" + if account_id in self._accounts: + return self._accounts[account_id] + self._accounts[account_id] = self._find_account(account_id) + return self._accounts[account_id] @staticmethod def _find_account(account_id: str) -> AdAccount: diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py new file mode 100644 index 0000000000000..c8b6c7e109a20 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py @@ -0,0 +1,82 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging +from typing import Any, List, Mapping + +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + +logger = logging.getLogger("airbyte_logger") + + +class MigrateAccountIdToArray: + """ + This class stands for migrating the config at runtime. + This migration is backwards compatible with the previous version, as new property will be created. + When falling back to the previous source version connector will use old property `account_id`. + + Starting from `1.3.0`, the `account_id` property is replaced with `account_ids` property, which is a list of strings. + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + migrate_from_key: str = "account_id" + migrate_to_key: str = "account_ids" + + @classmethod + def should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines whether the config should be migrated to have the new structure for the `custom_reports`, + based on the source spec. + Returns: + > True, if the transformation is necessary + > False, otherwise. + > Raises the Exception if the structure could not be migrated. + """ + return False if config.get(cls.migrate_to_key) else True + + @classmethod + def transform(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: + # transform the config + config[cls.migrate_to_key] = [config[cls.migrate_from_key]] + # return transformed config + return config + + @classmethod + def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: + # modify the config + migrated_config = cls.transform(config) + # save the config + source.write_config(migrated_config, config_path) + # return modified config + return migrated_config + + @classmethod + def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository._message_queue: + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: Source) -> None: + """ + This method checks the input args, should the config be migrated, + transform if neccessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls.should_migrate(config): + cls.emit_control_message( + cls.modify_and_save(config_path, source, config), + ) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/run.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/run.py index c99ffb1439061..2e92663e42fd8 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/run.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/run.py @@ -7,9 +7,11 @@ from airbyte_cdk.entrypoint import launch +from .config_migrations import MigrateAccountIdToArray from .source import SourceFacebookMarketing def run(): source = SourceFacebookMarketing() + MigrateAccountIdToArray.migrate(sys.argv[1:], source) launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/activities.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/activities.json index 4b5c729e0686d..69a31b5f8b553 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/activities.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/activities.json @@ -1,5 +1,8 @@ { "properties": { + "account_id": { + "type": ["null", "string"] + }, "actor_id": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/videos.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/videos.json index 9aa51674dd101..3a146978ada69 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/videos.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/videos.json @@ -1,5 +1,8 @@ { "properties": { + "account_id": { + "type": ["null", "string"] + }, "id": { "type": "string" }, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 3d900ec6d5192..65e8c057852a4 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -76,6 +76,9 @@ def _validate_and_transform(self, config: Mapping[str, Any]): if config.end_date: config.end_date = pendulum.instance(config.end_date) + + config.account_ids = list(config.account_ids) + return config def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: @@ -93,11 +96,20 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> if config.start_date and config.end_date < config.start_date: return False, "End date must be equal or after start date." - api = API(account_id=config.account_id, access_token=config.access_token, page_size=config.page_size) + api = API(access_token=config.access_token, page_size=config.page_size) + + for account_id in config.account_ids: + # Get Ad Account to check creds + logger.info(f"Attempting to retrieve information for account with ID: {account_id}") + ad_account = api.get_account(account_id=account_id) + logger.info(f"Successfully retrieved account information for account: {ad_account}") + + # make sure that we have valid combination of "action_breakdowns" and "breakdowns" parameters + for stream in self.get_custom_insights_streams(api, config): + stream.check_breakdowns(account_id=account_id) - # Get Ad Account to check creds - ad_account = api.account - logger.info(f"Select account {ad_account}") + except facebook_business.exceptions.FacebookRequestError as e: + return False, e._api_error_message except AirbyteTracedException as e: return False, f"{e.message}. Full error: {e.internal_message}" @@ -105,12 +117,6 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> except Exception as e: return False, f"Unexpected error: {repr(e)}" - # make sure that we have valid combination of "action_breakdowns" and "breakdowns" parameters - for stream in self.get_custom_insights_streams(api, config): - try: - stream.check_breakdowns() - except facebook_business.exceptions.FacebookRequestError as e: - return False, e._api_error_message return True, None def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: @@ -124,22 +130,24 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: config.start_date = validate_start_date(config.start_date) config.end_date = validate_end_date(config.start_date, config.end_date) - api = API(account_id=config.account_id, access_token=config.access_token, page_size=config.page_size) + api = API(access_token=config.access_token, page_size=config.page_size) # if start_date not specified then set default start_date for report streams to 2 years ago report_start_date = config.start_date or pendulum.now().add(years=-2) insights_args = dict( api=api, + account_ids=config.account_ids, start_date=report_start_date, end_date=config.end_date, insights_lookback_window=config.insights_lookback_window, insights_job_timeout=config.insights_job_timeout, ) streams = [ - AdAccount(api=api), + AdAccount(api=api, account_ids=config.account_ids), AdSets( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, @@ -147,6 +155,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: ), Ads( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, @@ -154,6 +163,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: ), AdCreatives( api=api, + account_ids=config.account_ids, fetch_thumbnail_images=config.fetch_thumbnail_images, page_size=config.page_size, ), @@ -179,6 +189,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: AdsInsightsDemographicsGender(page_size=config.page_size, **insights_args), Campaigns( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, @@ -186,16 +197,19 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: ), CustomConversions( api=api, + account_ids=config.account_ids, include_deleted=config.include_deleted, page_size=config.page_size, ), CustomAudiences( api=api, + account_ids=config.account_ids, include_deleted=config.include_deleted, page_size=config.page_size, ), Images( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, @@ -203,6 +217,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: ), Videos( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, @@ -210,6 +225,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: ), Activities( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, @@ -275,6 +291,7 @@ def get_custom_insights_streams(self, api: API, config: ConnectorConfig) -> List ) stream = AdsInsights( api=api, + account_ids=config.account_ids, name=f"Custom{insight.name}", fields=list(insight_fields), breakdowns=list(set(insight.breakdowns)), diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index be53cf51d84d7..951ce0a2a63c1 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -5,11 +5,11 @@ import logging from datetime import datetime, timezone from enum import Enum -from typing import List, Optional +from typing import List, Optional, Set from airbyte_cdk.sources.config import BaseConfig from facebook_business.adobjects.adsinsights import AdsInsights -from pydantic import BaseModel, Field, PositiveInt +from pydantic import BaseModel, Field, PositiveInt, constr logger = logging.getLogger("airbyte") @@ -112,18 +112,18 @@ class ConnectorConfig(BaseConfig): class Config: title = "Source Facebook Marketing" - account_id: str = Field( - title="Ad Account ID", + account_ids: Set[constr(regex="^[0-9]+$")] = Field( + title="Ad Account ID(s)", order=0, description=( - "The Facebook Ad account ID to use when pulling data from the Facebook Marketing API. " + "The Facebook Ad account ID(s) to pull data from. " "The Ad account ID number is in the account dropdown menu or in your browser's address " 'bar of your Meta Ads Manager. ' 'See the docs for more information.' ), - pattern="^[0-9]+$", - pattern_descriptor="1234567890", + pattern_descriptor="The Ad Account ID must be a number.", examples=["111111111111111"], + min_items=1, ) access_token: str = Field( diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py index 8bfcc6fe74afc..738507e4408bc 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py @@ -32,13 +32,14 @@ class InsightAsyncJobManager: # limit is not reliable indicator of async workload capability we still have to use this parameter. MAX_JOBS_IN_QUEUE = 100 - def __init__(self, api: "API", jobs: Iterator[AsyncJob]): + def __init__(self, api: "API", jobs: Iterator[AsyncJob], account_id: str): """Init :param api: :param jobs: """ self._api = api + self._account_id = account_id self._jobs = iter(jobs) self._running_jobs = [] @@ -147,4 +148,4 @@ def _update_api_throttle_limit(self): respond with empty list of data so api use "x-fb-ads-insights-throttle" header to update current insights throttle limit. """ - self._api.account.get_insights() + self._api.get_account(account_id=self._account_id).get_insights() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index eadef8012e5d7..c671a4b9b917b 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -11,7 +11,6 @@ from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from airbyte_cdk.utils import AirbyteTracedException -from cached_property import cached_property from facebook_business.exceptions import FacebookBadObjectError, FacebookRequestError from source_facebook_marketing.streams.async_job import AsyncJob, InsightAsyncJob from source_facebook_marketing.streams.async_job_manager import InsightAsyncJobManager @@ -70,7 +69,7 @@ def __init__( super().__init__(**kwargs) self._start_date = self._start_date.date() self._end_date = self._end_date.date() - self._fields = fields + self._custom_fields = fields if action_breakdowns_allow_empty: if action_breakdowns is not None: self.action_breakdowns = action_breakdowns @@ -87,9 +86,9 @@ def __init__( self.level = level # state - self._cursor_value: Optional[pendulum.Date] = None # latest period that was read - self._next_cursor_value = self._get_start_date() - self._completed_slices = set() + self._cursor_values: Optional[Mapping[str, pendulum.Date]] = None # latest period that was read for each account + self._next_cursor_values = self._get_start_date() + self._completed_slices = {account_id: set() for account_id in self._account_ids} @property def name(self) -> str: @@ -128,6 +127,8 @@ def read_records( ) -> Iterable[Mapping[str, Any]]: """Waits for current job to finish (slice) and yield its result""" job = stream_slice["insight_job"] + account_id = stream_slice["account_id"] + try: for obj in job.get_result(): data = obj.export_all_data() @@ -142,25 +143,30 @@ def read_records( except FacebookRequestError as exc: raise traced_exception(exc) - self._completed_slices.add(job.interval.start) - if job.interval.start == self._next_cursor_value: - self._advance_cursor() + self._completed_slices[account_id].add(job.interval.start) + if job.interval.start == self._next_cursor_values[account_id]: + self._advance_cursor(account_id) @property def state(self) -> MutableMapping[str, Any]: """State getter, the result can be stored by the source""" - if self._cursor_value: - return { - self.cursor_field: self._cursor_value.isoformat(), - "slices": [d.isoformat() for d in self._completed_slices], - "time_increment": self.time_increment, - } + new_state = {account_id: {} for account_id in self._account_ids} + + if self._cursor_values: + for account_id in self._account_ids: + if account_id in self._cursor_values and self._cursor_values[account_id]: + new_state[account_id] = {self.cursor_field: self._cursor_values[account_id].isoformat()} + + new_state[account_id]["slices"] = {d.isoformat() for d in self._completed_slices[account_id]} + new_state["time_increment"] = self.time_increment + return new_state if self._completed_slices: - return { - "slices": [d.isoformat() for d in self._completed_slices], - "time_increment": self.time_increment, - } + for account_id in self._account_ids: + new_state[account_id]["slices"] = {d.isoformat() for d in self._completed_slices[account_id]} + + new_state["time_increment"] = self.time_increment + return new_state return {} @@ -170,13 +176,23 @@ def state(self, value: Mapping[str, Any]): # if the time increment configured for this stream is different from the one in the previous state # then the previous state object is invalid and we should start replicating data from scratch # to achieve this, we skip setting the state - if value.get("time_increment", 1) != self.time_increment: + transformed_state = self._transform_state_from_old_format(value, ["time_increment"]) + if transformed_state.get("time_increment", 1) != self.time_increment: logger.info(f"Ignoring bookmark for {self.name} because of different `time_increment` option.") return - self._cursor_value = pendulum.parse(value[self.cursor_field]).date() if value.get(self.cursor_field) else None - self._completed_slices = set(pendulum.parse(v).date() for v in value.get("slices", [])) - self._next_cursor_value = self._get_start_date() + self._cursor_values = { + account_id: pendulum.parse(transformed_state[account_id][self.cursor_field]).date() + if transformed_state.get(account_id, {}).get(self.cursor_field) + else None + for account_id in self._account_ids + } + self._completed_slices = { + account_id: set(pendulum.parse(v).date() for v in transformed_state.get(account_id, {}).get("slices", [])) + for account_id in self._account_ids + } + + self._next_cursor_values = self._get_start_date() def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): """Update stream state from latest record @@ -186,40 +202,47 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late """ return self.state - def _date_intervals(self) -> Iterator[pendulum.Date]: + def _date_intervals(self, account_id: str) -> Iterator[pendulum.Date]: """Get date period to sync""" - if self._end_date < self._next_cursor_value: + if self._end_date < self._next_cursor_values[account_id]: return - date_range = self._end_date - self._next_cursor_value + date_range = self._end_date - self._next_cursor_values[account_id] yield from date_range.range("days", self.time_increment) - def _advance_cursor(self): + def _advance_cursor(self, account_id: str): """Iterate over state, find continuing sequence of slices. Get last value, advance cursor there and remove slices from state""" - for ts_start in self._date_intervals(): - if ts_start not in self._completed_slices: - self._next_cursor_value = ts_start + for ts_start in self._date_intervals(account_id): + if ts_start not in self._completed_slices[account_id]: + self._next_cursor_values[account_id] = ts_start break - self._completed_slices.remove(ts_start) - self._cursor_value = ts_start + self._completed_slices[account_id].remove(ts_start) + if self._cursor_values: + self._cursor_values[account_id] = ts_start + else: + self._cursor_values = {account_id: ts_start} - def _generate_async_jobs(self, params: Mapping) -> Iterator[AsyncJob]: + def _generate_async_jobs(self, params: Mapping, account_id: str) -> Iterator[AsyncJob]: """Generator of async jobs :param params: :return: """ - self._next_cursor_value = self._get_start_date() - for ts_start in self._date_intervals(): - if ts_start in self._completed_slices: + self._next_cursor_values = self._get_start_date() + for ts_start in self._date_intervals(account_id): + if ts_start in self._completed_slices.get(account_id, []): continue ts_end = ts_start + pendulum.duration(days=self.time_increment - 1) interval = pendulum.Period(ts_start, ts_end) yield InsightAsyncJob( - api=self._api.api, edge_object=self._api.account, interval=interval, params=params, job_timeout=self.insights_job_timeout + api=self._api.api, + edge_object=self._api.get_account(account_id=account_id), + interval=interval, + params=params, + job_timeout=self.insights_job_timeout, ) - def check_breakdowns(self): + def check_breakdowns(self, account_id: str): """ Making call to check "action_breakdowns" and "breakdowns" combinations https://developers.facebook.com/docs/marketing-api/insights/breakdowns#combiningbreakdowns @@ -229,7 +252,7 @@ def check_breakdowns(self): "breakdowns": self.breakdowns, "fields": ["account_id"], } - self._api.account.get_insights(params=params, is_async=False) + self._api.get_account(account_id=account_id).get_insights(params=params, is_async=False) def _response_data_is_valid(self, data: Iterable[Mapping[str, Any]]) -> bool: """ @@ -255,14 +278,19 @@ def stream_slices( if stream_state: self.state = stream_state - try: - manager = InsightAsyncJobManager(api=self._api, jobs=self._generate_async_jobs(params=self.request_params())) - for job in manager.completed_jobs(): - yield {"insight_job": job} - except FacebookRequestError as exc: - raise traced_exception(exc) + for account_id in self._account_ids: + try: + manager = InsightAsyncJobManager( + api=self._api, + jobs=self._generate_async_jobs(params=self.request_params(), account_id=account_id), + account_id=account_id, + ) + for job in manager.completed_jobs(): + yield {"insight_job": job, "account_id": account_id} + except FacebookRequestError as exc: + raise traced_exception(exc) - def _get_start_date(self) -> pendulum.Date: + def _get_start_date(self) -> Mapping[str, pendulum.Date]: """Get start date to begin sync with. It is not that trivial as it might seem. There are few rules: - don't read data older than start_date @@ -277,33 +305,42 @@ def _get_start_date(self) -> pendulum.Date: today = pendulum.today().date() oldest_date = today - self.INSIGHTS_RETENTION_PERIOD refresh_date = today - self.insights_lookback_period - if self._cursor_value: - start_date = self._cursor_value + pendulum.duration(days=self.time_increment) - if start_date > refresh_date: - logger.info( - f"The cursor value within refresh period ({self.insights_lookback_period}), start sync from {refresh_date} instead." + + start_dates_for_account = {} + for account_id in self._account_ids: + cursor_value = self._cursor_values.get(account_id) if self._cursor_values else None + if cursor_value: + start_date = cursor_value + pendulum.duration(days=self.time_increment) + if start_date > refresh_date: + logger.info( + f"The cursor value within refresh period ({self.insights_lookback_period}), start sync from {refresh_date} instead." + ) + start_date = min(start_date, refresh_date) + + if start_date < self._start_date: + logger.warning(f"Ignore provided state and start sync from start_date ({self._start_date}).") + start_date = max(start_date, self._start_date) + else: + start_date = self._start_date + if start_date < oldest_date: + logger.warning( + f"Loading insights older then {self.INSIGHTS_RETENTION_PERIOD} is not possible. Start sync from {oldest_date}." ) - start_date = min(start_date, refresh_date) + start_dates_for_account[account_id] = max(oldest_date, start_date) - if start_date < self._start_date: - logger.warning(f"Ignore provided state and start sync from start_date ({self._start_date}).") - start_date = max(start_date, self._start_date) - else: - start_date = self._start_date - if start_date < oldest_date: - logger.warning(f"Loading insights older then {self.INSIGHTS_RETENTION_PERIOD} is not possible. Start sync from {oldest_date}.") - return max(oldest_date, start_date) + return start_dates_for_account def request_params(self, **kwargs) -> MutableMapping[str, Any]: - return { + req_params = { "level": self.level, "action_breakdowns": self.action_breakdowns, "action_report_time": self.action_report_time, "breakdowns": self.breakdowns, - "fields": self.fields, + "fields": self.fields(), "time_increment": self.time_increment, "action_attribution_windows": self.action_attribution_windows, } + return req_params def _state_filter(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]: """Works differently for insights, so remove it""" @@ -315,19 +352,23 @@ def get_json_schema(self) -> Mapping[str, Any]: """ loader = ResourceSchemaLoader(package_name_from_class(self.__class__)) schema = loader.get_schema("ads_insights") - if self._fields: + if self._custom_fields: # 'date_stop' and 'account_id' are also returned by default, even if they are not requested - custom_fields = set(self._fields + [self.cursor_field, "date_stop", "account_id", "ad_id"]) + custom_fields = set(self._custom_fields + [self.cursor_field, "date_stop", "account_id", "ad_id"]) schema["properties"] = {k: v for k, v in schema["properties"].items() if k in custom_fields} if self.breakdowns: breakdowns_properties = loader.get_schema("ads_insights_breakdowns")["properties"] schema["properties"].update({prop: breakdowns_properties[prop] for prop in self.breakdowns}) return schema - @cached_property - def fields(self) -> List[str]: + def fields(self, **kwargs) -> List[str]: """List of fields that we want to query, for now just all properties from stream's schema""" + if self._custom_fields: + return self._custom_fields + if self._fields: return self._fields + schema = ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ads_insights") - return list(schema.get("properties", {}).keys()) + self._fields = list(schema.get("properties", {}).keys()) + return self._fields diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py index 01b8488ca4ba7..9f396077df8a1 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py @@ -12,7 +12,6 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from cached_property import cached_property from facebook_business.adobjects.abstractobject import AbstractObject from facebook_business.exceptions import FacebookRequestError from source_facebook_marketing.streams.common import traced_exception @@ -43,16 +42,20 @@ class FBMarketingStream(Stream, ABC): def availability_strategy(self) -> Optional["AvailabilityStrategy"]: return None - def __init__(self, api: "API", include_deleted: bool = False, page_size: int = 100, **kwargs): + def __init__(self, api: "API", account_ids: List[str], include_deleted: bool = False, page_size: int = 100, **kwargs): super().__init__(**kwargs) self._api = api + self._account_ids = account_ids self.page_size = page_size if page_size is not None else 100 self._include_deleted = include_deleted if self.enable_deleted else False + self._fields = None - @cached_property - def fields(self) -> List[str]: + def fields(self, **kwargs) -> List[str]: """List of fields that we want to query, for now just all properties from stream's schema""" - return list(self.get_json_schema().get("properties", {}).keys()) + if self._fields: + return self._fields + self._saved_fields = list(self.get_json_schema().get("properties", {}).keys()) + return self._saved_fields @classmethod def fix_date_time(cls, record): @@ -78,6 +81,70 @@ def fix_date_time(cls, record): for entry in record: cls.fix_date_time(entry) + @staticmethod + def add_account_id(record, account_id: str): + if "account_id" not in record: + record["account_id"] = account_id + + def get_account_state(self, account_id: str, stream_state: Mapping[str, Any] = None) -> MutableMapping[str, Any]: + """ + Retrieve the state for a specific account. + + If multiple account IDs are present, the state for the specific account ID + is returned if it exists in the stream state. If only one account ID is + present, the entire stream state is returned. + + :param account_id: The account ID for which to retrieve the state. + :param stream_state: The current stream state, optional. + :return: The state information for the specified account as a MutableMapping. + """ + if stream_state and account_id and account_id in stream_state: + account_state = stream_state.get(account_id) + + # copy `include_deleted` from general stream state + if "include_deleted" in stream_state: + account_state["include_deleted"] = stream_state["include_deleted"] + return account_state + elif len(self._account_ids) == 1: + return stream_state + else: + return {} + + def _transform_state_from_old_format(self, state: Mapping[str, Any], move_fields: List[str] = None) -> Mapping[str, Any]: + """ + Transforms the state from an old format to a new format based on account IDs. + + This method transforms the old state to be a dictionary where the keys are account IDs. + If the state is in the old format (not keyed by account IDs), it will transform the state + by nesting it under the account ID. + + :param state: The original state dictionary to transform. + :param move_fields: A list of field names whose values should be moved to the top level of the new state dictionary. + :return: The transformed state dictionary. + """ + + # If the state already contains any of the account IDs, return the state as is. + for account_id in self._account_ids: + if account_id in state: + return state + + # Handle the case where there is only one account ID. + # Transform the state by nesting it under the account ID. + if state and len(self._account_ids) == 1: + account_id = self._account_ids[0] + new_state = {account_id: state} + + # Move specified fields to the top level of the new state. + if move_fields: + for move_field in move_fields: + if move_field in state: + new_state[move_field] = state.pop(move_field) + + return new_state + + # If the state is empty or there are multiple account IDs, return an empty dictionary. + return {} + def read_records( self, sync_mode: SyncMode, @@ -86,15 +153,24 @@ def read_records( stream_state: Mapping[str, Any] = None, ) -> Iterable[Mapping[str, Any]]: """Main read method used by CDK""" + account_id = stream_slice["account_id"] + account_state = stream_slice.get("stream_state", {}) + try: - for record in self.list_objects(params=self.request_params(stream_state=stream_state)): + for record in self.list_objects(params=self.request_params(stream_state=account_state), account_id=account_id): if isinstance(record, AbstractObject): record = record.export_all_data() # convert FB object to dict self.fix_date_time(record) + self.add_account_id(record, stream_slice["account_id"]) yield record except FacebookRequestError as exc: raise traced_exception(exc) + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + for account_id in self._account_ids: + account_state = self.get_account_state(account_id, stream_state) + yield {"account_id": account_id, "stream_state": account_state} + @abstractmethod def list_objects(self, params: Mapping[str, Any]) -> Iterable: """List FB objects, these objects will be loaded in read_records later with their details. @@ -150,17 +226,21 @@ def __init__(self, start_date: Optional[datetime], end_date: Optional[datetime], def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): """Update stream state from latest record""" - potentially_new_records_in_the_past = self._include_deleted and not current_stream_state.get("include_deleted", False) + account_id = latest_record["account_id"] + state_for_accounts = self._transform_state_from_old_format(current_stream_state, ["include_deleted"]) + account_state = self.get_account_state(account_id, state_for_accounts) + + potentially_new_records_in_the_past = self._include_deleted and not account_state.get("include_deleted", False) record_value = latest_record[self.cursor_field] - state_value = current_stream_state.get(self.cursor_field) or record_value + state_value = account_state.get(self.cursor_field) or record_value max_cursor = max(pendulum.parse(state_value), pendulum.parse(record_value)) if potentially_new_records_in_the_past: max_cursor = record_value - return { - self.cursor_field: str(max_cursor), - "include_deleted": self._include_deleted, - } + state_for_accounts.setdefault(account_id, {})[self.cursor_field] = str(max_cursor) + + state_for_accounts["include_deleted"] = self._include_deleted + return state_for_accounts def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: """Include state filter""" @@ -207,28 +287,31 @@ class FBMarketingReversedIncrementalStream(FBMarketingIncrementalStream, ABC): def __init__(self, **kwargs): super().__init__(**kwargs) - self._cursor_value = None - self._max_cursor_value = None + self._cursor_values = {} @property def state(self) -> Mapping[str, Any]: """State getter, get current state and serialize it to emmit Airbyte STATE message""" - if self._cursor_value: - return { - self.cursor_field: self._cursor_value, - "include_deleted": self._include_deleted, - } + if self._cursor_values: + result_state = {account_id: {self.cursor_field: cursor_value} for account_id, cursor_value in self._cursor_values.items()} + result_state["include_deleted"] = self._include_deleted + return result_state return {} @state.setter def state(self, value: Mapping[str, Any]): """State setter, ignore state if current settings mismatch saved state""" - if self._include_deleted and not value.get("include_deleted"): + transformed_state = self._transform_state_from_old_format(value, ["include_deleted"]) + if self._include_deleted and not transformed_state.get("include_deleted"): logger.info(f"Ignoring bookmark for {self.name} because of enabled `include_deleted` option") return - self._cursor_value = pendulum.parse(value[self.cursor_field]) + self._cursor_values = {} + for account_id in self._account_ids: + cursor_value = transformed_state.get(account_id, {}).get(self.cursor_field) + if cursor_value is not None: + self._cursor_values[account_id] = pendulum.parse(cursor_value) def _state_filter(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]: """Don't have classic cursor filtering""" @@ -250,20 +333,27 @@ def read_records( - update state only when we reach the end - stop reading when we reached the end """ + account_id = stream_slice["account_id"] + account_state = stream_slice.get("stream_state") + try: - records_iter = self.list_objects(params=self.request_params(stream_state=stream_state)) + records_iter = self.list_objects(params=self.request_params(stream_state=account_state), account_id=account_id) + account_cursor = self._cursor_values.get(account_id) + + max_cursor_value = None for record in records_iter: record_cursor_value = pendulum.parse(record[self.cursor_field]) - if self._cursor_value and record_cursor_value < self._cursor_value: + if account_cursor and record_cursor_value < account_cursor: break if not self._include_deleted and self.get_record_deleted_status(record): continue - self._max_cursor_value = max(self._max_cursor_value, record_cursor_value) if self._max_cursor_value else record_cursor_value + max_cursor_value = max(max_cursor_value, record_cursor_value) if max_cursor_value else record_cursor_value record = record.export_all_data() self.fix_date_time(record) + self.add_account_id(record, stream_slice["account_id"]) yield record - self._cursor_value = self._max_cursor_value + self._cursor_values[account_id] = max_cursor_value except FacebookRequestError as exc: raise traced_exception(exc) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py index 23fd4b565bdf1..c7fd0237963bd 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py @@ -9,7 +9,6 @@ import pendulum import requests from airbyte_cdk.models import SyncMode -from cached_property import cached_property from facebook_business.adobjects.adaccount import AdAccount as FBAdAccount from facebook_business.adobjects.adimage import AdImage from facebook_business.adobjects.user import User @@ -48,10 +47,13 @@ def __init__(self, fetch_thumbnail_images: bool = False, **kwargs): super().__init__(**kwargs) self._fetch_thumbnail_images = fetch_thumbnail_images - @cached_property - def fields(self) -> List[str]: + def fields(self, **kwargs) -> List[str]: """Remove "thumbnail_data_url" field because it is computed field and it's not a field that we can request from Facebook""" - return [f for f in super().fields if f != "thumbnail_data_url"] + if self._fields: + return self._fields + + self._fields = [f for f in super().fields(**kwargs) if f != "thumbnail_data_url"] + return self._fields def read_records( self, @@ -68,8 +70,8 @@ def read_records( record["thumbnail_data_url"] = fetch_thumbnail_data_url(thumbnail_url) yield record - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_ad_creatives(params=params, fields=self.fields) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_ad_creatives(params=params, fields=self.fields()) class CustomConversions(FBMarketingStream): @@ -78,8 +80,8 @@ class CustomConversions(FBMarketingStream): entity_prefix = "customconversion" enable_deleted = False - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_custom_conversions(params=params, fields=self.fields) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_custom_conversions(params=params, fields=self.fields()) class CustomAudiences(FBMarketingStream): @@ -91,8 +93,8 @@ class CustomAudiences(FBMarketingStream): # https://github.com/airbytehq/oncall/issues/2765 fields_exceptions = ["rule"] - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_custom_audiences(params=params, fields=self.fields) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_custom_audiences(params=params, fields=self.fields()) class Ads(FBMarketingIncrementalStream): @@ -100,8 +102,8 @@ class Ads(FBMarketingIncrementalStream): entity_prefix = "ad" - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_ads(params=params, fields=self.fields) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_ads(params=params, fields=self.fields()) class AdSets(FBMarketingIncrementalStream): @@ -109,8 +111,8 @@ class AdSets(FBMarketingIncrementalStream): entity_prefix = "adset" - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_ad_sets(params=params, fields=self.fields) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_ad_sets(params=params, fields=self.fields()) class Campaigns(FBMarketingIncrementalStream): @@ -118,8 +120,8 @@ class Campaigns(FBMarketingIncrementalStream): entity_prefix = "campaign" - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_campaigns(params=params, fields=self.fields) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_campaigns(params=params, fields=self.fields()) class Activities(FBMarketingIncrementalStream): @@ -129,8 +131,16 @@ class Activities(FBMarketingIncrementalStream): cursor_field = "event_time" primary_key = None - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_activities(fields=self.fields, params=params) + def fields(self, **kwargs) -> List[str]: + """Remove account_id from fields as cannot be requested, but it is part of schema as foreign key, will be added during processing""" + if self._fields: + return self._fields + + self._fields = [f for f in super().fields(**kwargs) if f != "account_id"] + return self._fields + + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_activities(fields=self.fields(), params=params) def _state_filter(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]: """Additional filters associated with state if any set""" @@ -160,9 +170,17 @@ class Videos(FBMarketingReversedIncrementalStream): entity_prefix = "video" - def list_objects(self, params: Mapping[str, Any]) -> Iterable: + def fields(self, **kwargs) -> List[str]: + """Remove account_id from fields as cannot be requested, but it is part of schema as foreign key, will be added during processing""" + if self._fields: + return self._fields + + self._fields = [f for f in super().fields() if f != "account_id"] + return self._fields + + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: # Remove filtering as it is not working for this stream since 2023-01-13 - return self._api.account.get_ad_videos(params=params, fields=self.fields) + return self._api.get_account(account_id=account_id).get_ad_videos(params=params, fields=self.fields()) class AdAccount(FBMarketingStream): @@ -171,55 +189,66 @@ class AdAccount(FBMarketingStream): use_batch = False enable_deleted = False - def get_task_permissions(self) -> Set[str]: + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._fields_dict = {} + + def get_task_permissions(self, account_id: str) -> Set[str]: """https://developers.facebook.com/docs/marketing-api/reference/ad-account/assigned_users/""" res = set() me = User(fbid="me", api=self._api.api) for business_user in me.get_business_users(): - assigned_users = self._api.account.get_assigned_users(params={"business": business_user["business"].get_id()}) + assigned_users = self._api.get_account(account_id=account_id).get_assigned_users( + params={"business": business_user["business"].get_id()} + ) for assigned_user in assigned_users: if business_user.get_id() == assigned_user.get_id(): res.update(set(assigned_user["tasks"])) return res - @cached_property - def fields(self) -> List[str]: - properties = super().fields + def fields(self, account_id: str, **kwargs) -> List[str]: + if self._fields_dict.get(account_id): + return self._fields_dict.get(account_id) + + properties = super().fields(**kwargs) # https://developers.facebook.com/docs/marketing-apis/guides/javascript-ads-dialog-for-payments/ # To access "funding_source_details", the user making the API call must have a MANAGE task permission for # that specific ad account. - permissions = self.get_task_permissions() + permissions = self.get_task_permissions(account_id=account_id) if "funding_source_details" in properties and "MANAGE" not in permissions: properties.remove("funding_source_details") if "is_prepay_account" in properties and "MANAGE" not in permissions: properties.remove("is_prepay_account") + + self._fields_dict[account_id] = properties return properties - def list_objects(self, params: Mapping[str, Any]) -> Iterable: + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: """noop in case of AdAccount""" - fields = self.fields + fields = self.fields(account_id=account_id) try: - return [FBAdAccount(self._api.account.get_id()).api_get(fields=fields)] + print(f"{self._api.get_account(account_id=account_id).get_id()=} {account_id=}") + return [FBAdAccount(self._api.get_account(account_id=account_id).get_id()).api_get(fields=fields)] except FacebookRequestError as e: # This is a workaround for cases when account seem to have all the required permissions # but despite of that is not allowed to get `owner` field. See (https://github.com/airbytehq/oncall/issues/3167) if e.api_error_code() == 200 and e.api_error_message() == "(#200) Requires business_management permission to manage the object": fields.remove("owner") - return [FBAdAccount(self._api.account.get_id()).api_get(fields=fields)] + return [FBAdAccount(self._api.get_account(account_id=account_id).get_id()).api_get(fields=fields)] # FB api returns a non-obvious error when accessing the `funding_source_details` field # even though user is granted all the required permissions (`MANAGE`) # https://github.com/airbytehq/oncall/issues/3031 if e.api_error_code() == 100 and e.api_error_message() == "Unsupported request - method type: get": fields.remove("funding_source_details") - return [FBAdAccount(self._api.account.get_id()).api_get(fields=fields)] + return [FBAdAccount(self._api.get_account(account_id=account_id).get_id()).api_get(fields=fields)] raise e class Images(FBMarketingReversedIncrementalStream): """See: https://developers.facebook.com/docs/marketing-api/reference/ad-image""" - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_ad_images(params=params, fields=self.fields) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_ad_images(params=params, fields=self.fields(account_id=account_id)) def get_record_deleted_status(self, record) -> bool: return record[AdImage.Field.status] == AdImage.Status.deleted diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py index ad2454b02ea43..a7574ce206f94 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py @@ -23,7 +23,7 @@ def account_id_fixture(): @fixture(scope="session", name="some_config") def some_config_fixture(account_id): - return {"start_date": "2021-01-23T00:00:00Z", "account_id": f"{account_id}", "access_token": "unknown_token"} + return {"start_date": "2021-01-23T00:00:00Z", "account_ids": [f"{account_id}"], "access_token": "unknown_token"} @fixture(autouse=True) @@ -49,8 +49,10 @@ def fb_account_response_fixture(account_id): @fixture(name="api") def api_fixture(some_config, requests_mock, fb_account_response): - api = API(account_id=some_config["account_id"], access_token=some_config["access_token"], page_size=100) + api = API(access_token=some_config["access_token"], page_size=100) requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/me/adaccounts", [fb_account_response]) - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{some_config['account_id']}/", [fb_account_response]) + requests_mock.register_uri( + "GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{some_config['account_ids'][0]}/", [fb_account_response] + ) return api diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py index 3bc8a37c2db8d..29b2ccbfaaffd 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py @@ -136,6 +136,6 @@ def test__handle_call_rate_limit(self, mocker, fb_api, params, min_rate, usage, def test_find_account(self, api, account_id, requests_mock): requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/", [{"json": {"id": "act_test"}}]) - account = api._find_account(account_id) + account = api.get_account(account_id) assert isinstance(account, AdAccount) assert account.get_id() == "act_test" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job_manager.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job_manager.py index a9234fc31465a..cb0cffffeabbb 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job_manager.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job_manager.py @@ -29,24 +29,24 @@ def update_job_mock_fixture(mocker): class TestInsightAsyncManager: - def test_jobs_empty(self, api): + def test_jobs_empty(self, api, some_config): """Should work event without jobs""" - manager = InsightAsyncJobManager(api=api, jobs=[]) + manager = InsightAsyncJobManager(api=api, jobs=[], account_id=some_config["account_ids"][0]) jobs = list(manager.completed_jobs()) assert not jobs - def test_jobs_completed_immediately(self, api, mocker, time_mock): + def test_jobs_completed_immediately(self, api, mocker, time_mock, some_config): """Manager should emmit jobs without waiting if they completed""" jobs = [ mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False), mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False), ] - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager(api=api, jobs=jobs, account_id=some_config["account_ids"][0]) completed_jobs = list(manager.completed_jobs()) assert jobs == completed_jobs time_mock.sleep.assert_not_called() - def test_jobs_wait(self, api, mocker, time_mock, update_job_mock): + def test_jobs_wait(self, api, mocker, time_mock, update_job_mock, some_config): """Manager should return completed jobs and wait for others""" def update_job_behaviour(): @@ -61,7 +61,7 @@ def update_job_behaviour(): mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False), mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False), ] - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager(api=api, jobs=jobs, account_id=some_config["account_ids"][0]) job = next(manager.completed_jobs(), None) assert job == jobs[1] @@ -74,7 +74,7 @@ def update_job_behaviour(): job = next(manager.completed_jobs(), None) assert job is None - def test_job_restarted(self, api, mocker, time_mock, update_job_mock): + def test_job_restarted(self, api, mocker, time_mock, update_job_mock, some_config): """Manager should restart failed jobs""" def update_job_behaviour(): @@ -89,7 +89,7 @@ def update_job_behaviour(): mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True), mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False), ] - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager(api=api, jobs=jobs, account_id=some_config["account_ids"][0]) job = next(manager.completed_jobs(), None) assert job == jobs[0] @@ -101,7 +101,7 @@ def update_job_behaviour(): job = next(manager.completed_jobs(), None) assert job is None - def test_job_split(self, api, mocker, time_mock, update_job_mock): + def test_job_split(self, api, mocker, time_mock, update_job_mock, some_config): """Manager should split failed jobs when they fail second time""" def update_job_behaviour(): @@ -121,7 +121,7 @@ def update_job_behaviour(): sub_jobs[0].get_result.return_value = [1, 2] sub_jobs[1].get_result.return_value = [3, 4] jobs[1].split_job.return_value = sub_jobs - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager(api=api, jobs=jobs, account_id=some_config["account_ids"][0]) job = next(manager.completed_jobs(), None) assert job == jobs[0] @@ -134,7 +134,7 @@ def update_job_behaviour(): job = next(manager.completed_jobs(), None) assert job is None - def test_job_failed_too_many_times(self, api, mocker, time_mock, update_job_mock): + def test_job_failed_too_many_times(self, api, mocker, time_mock, update_job_mock, some_config): """Manager should fail when job failed too many times""" def update_job_behaviour(): @@ -147,12 +147,12 @@ def update_job_behaviour(): mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True), mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False), ] - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager(api=api, jobs=jobs, account_id=some_config["account_ids"][0]) with pytest.raises(JobException, match=f"{jobs[1]}: failed more than {InsightAsyncJobManager.MAX_NUMBER_OF_ATTEMPTS} times."): next(manager.completed_jobs(), None) - def test_nested_job_failed_too_many_times(self, api, mocker, time_mock, update_job_mock): + def test_nested_job_failed_too_many_times(self, api, mocker, time_mock, update_job_mock, some_config): """Manager should fail when a nested job within a ParentAsyncJob failed too many times""" def update_job_behaviour(): @@ -170,7 +170,7 @@ def update_job_behaviour(): mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True), mocker.Mock(spec=ParentAsyncJob, _jobs=sub_jobs, attempt_number=1, failed=False, completed=False), ] - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager(api=api, jobs=jobs, account_id=some_config["account_ids"][0]) with pytest.raises(JobException): next(manager.completed_jobs(), None) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py index 6f98004bcfce5..3d6ef2aaa5e69 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py @@ -48,8 +48,14 @@ def async_job_mock_fixture(mocker): class TestBaseInsightsStream: - def test_init(self, api): - stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) + def test_init(self, api, some_config): + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=datetime(2010, 1, 1), + end_date=datetime(2011, 1, 1), + insights_lookback_window=28, + ) assert not stream.breakdowns assert stream.action_breakdowns == ["action_type", "action_target_id", "action_destination"] @@ -57,9 +63,10 @@ def test_init(self, api): assert stream.primary_key == ["date_start", "account_id", "ad_id"] assert stream.action_report_time == "mixed" - def test_init_override(self, api): + def test_init_override(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), name="CustomName", @@ -73,7 +80,7 @@ def test_init_override(self, api): assert stream.name == "custom_name" assert stream.primary_key == ["date_start", "account_id", "ad_id", "test1", "test2"] - def test_read_records_all(self, mocker, api): + def test_read_records_all(self, mocker, api, some_config): """1. yield all from mock 2. if read slice 2, 3 state not changed if read slice 2, 3, 1 state changed to 3 @@ -83,6 +90,7 @@ def test_read_records_all(self, mocker, api): job.interval = pendulum.Period(pendulum.date(2010, 1, 1), pendulum.date(2010, 1, 1)) stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28, @@ -91,13 +99,13 @@ def test_read_records_all(self, mocker, api): records = list( stream.read_records( sync_mode=SyncMode.incremental, - stream_slice={"insight_job": job}, + stream_slice={"insight_job": job, "account_id": some_config["account_ids"][0]}, ) ) assert len(records) == 3 - def test_read_records_random_order(self, mocker, api): + def test_read_records_random_order(self, mocker, api, some_config): """1. yield all from mock 2. if read slice 2, 3 state not changed if read slice 2, 3, 1 state changed to 3 @@ -105,62 +113,144 @@ def test_read_records_random_order(self, mocker, api): job = mocker.Mock(spec=AsyncJob) job.get_result.return_value = [mocker.Mock(), mocker.Mock(), mocker.Mock()] job.interval = pendulum.Period(pendulum.date(2010, 1, 1), pendulum.date(2010, 1, 1)) - stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=datetime(2010, 1, 1), + end_date=datetime(2011, 1, 1), + insights_lookback_window=28, + ) records = list( stream.read_records( sync_mode=SyncMode.incremental, - stream_slice={"insight_job": job}, + stream_slice={"insight_job": job, "account_id": some_config["account_ids"][0]}, ) ) assert len(records) == 3 @pytest.mark.parametrize( - "state", + "state,result_state", [ - { - AdsInsights.cursor_field: "2010-10-03", - "slices": [ - "2010-01-01", - "2010-01-02", - ], - "time_increment": 1, - }, - { - AdsInsights.cursor_field: "2010-10-03", - }, - { - "slices": [ - "2010-01-01", - "2010-01-02", - ] - }, + # Old format + ( + { + AdsInsights.cursor_field: "2010-10-03", + "slices": [ + "2010-01-01", + "2010-01-02", + ], + "time_increment": 1, + }, + { + "unknown_account": { + AdsInsights.cursor_field: "2010-10-03", + "slices": { + "2010-01-01", + "2010-01-02", + }, + }, + "time_increment": 1, + }, + ), + ( + { + AdsInsights.cursor_field: "2010-10-03", + }, + { + "unknown_account": { + AdsInsights.cursor_field: "2010-10-03", + } + }, + ), + ( + { + "slices": [ + "2010-01-01", + "2010-01-02", + ] + }, + { + "unknown_account": { + "slices": { + "2010-01-01", + "2010-01-02", + } + } + }, + ), + # New format - nested with account_id + ( + { + "unknown_account": { + AdsInsights.cursor_field: "2010-10-03", + "slices": { + "2010-01-01", + "2010-01-02", + }, + }, + "time_increment": 1, + }, + None, + ), + ( + { + "unknown_account": { + AdsInsights.cursor_field: "2010-10-03", + } + }, + None, + ), + ( + { + "unknown_account": { + "slices": { + "2010-01-01", + "2010-01-02", + } + } + }, + None, + ), ], ) - def test_state(self, api, state): + def test_state(self, api, state, result_state, some_config): """State setter/getter should work with all combinations""" - stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=datetime(2010, 1, 1), + end_date=datetime(2011, 1, 1), + insights_lookback_window=28, + ) - assert stream.state == {} + assert stream.state == {"time_increment": 1, "unknown_account": {"slices": set()}} stream.state = state actual_state = stream.state - actual_state["slices"] = sorted(actual_state.get("slices", [])) - state["slices"] = sorted(state.get("slices", [])) - state["time_increment"] = 1 - assert actual_state == state + result_state = state if not result_state else result_state + result_state[some_config["account_ids"][0]]["slices"] = result_state[some_config["account_ids"][0]].get("slices", set()) + result_state["time_increment"] = 1 - def test_stream_slices_no_state(self, api, async_manager_mock, start_date): + assert actual_state == result_state + + def test_stream_slices_no_state(self, api, async_manager_mock, start_date, some_config): """Stream will use start_date when there is not state""" end_date = start_date + duration(weeks=2) - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) + stream = AdsInsights( + api=api, account_ids=some_config["account_ids"], start_date=start_date, end_date=end_date, insights_lookback_window=28 + ) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list(stream.stream_slices(stream_state=None, sync_mode=SyncMode.incremental)) - assert slices == [{"insight_job": 1}, {"insight_job": 2}, {"insight_job": 3}] + assert slices == [ + {"account_id": "unknown_account", "insight_job": 1}, + {"account_id": "unknown_account", "insight_job": 2}, + {"account_id": "unknown_account", "insight_job": 3}, + ] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) @@ -168,16 +258,22 @@ def test_stream_slices_no_state(self, api, async_manager_mock, start_date): assert generated_jobs[0].interval.start == start_date.date() assert generated_jobs[1].interval.start == start_date.date() + duration(days=1) - def test_stream_slices_no_state_close_to_now(self, api, async_manager_mock, recent_start_date): + def test_stream_slices_no_state_close_to_now(self, api, async_manager_mock, recent_start_date, some_config): """Stream will use start_date when there is not state and start_date within 28d from now""" start_date = recent_start_date end_date = pendulum.now() - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) + stream = AdsInsights( + api=api, account_ids=some_config["account_ids"], start_date=start_date, end_date=end_date, insights_lookback_window=28 + ) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list(stream.stream_slices(stream_state=None, sync_mode=SyncMode.incremental)) - assert slices == [{"insight_job": 1}, {"insight_job": 2}, {"insight_job": 3}] + assert slices == [ + {"account_id": "unknown_account", "insight_job": 1}, + {"account_id": "unknown_account", "insight_job": 2}, + {"account_id": "unknown_account", "insight_job": 3}, + ] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) @@ -185,17 +281,23 @@ def test_stream_slices_no_state_close_to_now(self, api, async_manager_mock, rece assert generated_jobs[0].interval.start == start_date.date() assert generated_jobs[1].interval.start == start_date.date() + duration(days=1) - def test_stream_slices_with_state(self, api, async_manager_mock, start_date): + def test_stream_slices_with_state(self, api, async_manager_mock, start_date, some_config): """Stream will use cursor_value from state when there is state""" end_date = start_date + duration(days=10) cursor_value = start_date + duration(days=5) state = {AdsInsights.cursor_field: cursor_value.date().isoformat()} - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) + stream = AdsInsights( + api=api, account_ids=some_config["account_ids"], start_date=start_date, end_date=end_date, insights_lookback_window=28 + ) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list(stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental)) - assert slices == [{"insight_job": 1}, {"insight_job": 2}, {"insight_job": 3}] + assert slices == [ + {"account_id": "unknown_account", "insight_job": 1}, + {"account_id": "unknown_account", "insight_job": 2}, + {"account_id": "unknown_account", "insight_job": 3}, + ] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) @@ -203,18 +305,24 @@ def test_stream_slices_with_state(self, api, async_manager_mock, start_date): assert generated_jobs[0].interval.start == cursor_value.date() + duration(days=1) assert generated_jobs[1].interval.start == cursor_value.date() + duration(days=2) - def test_stream_slices_with_state_close_to_now(self, api, async_manager_mock, recent_start_date): + def test_stream_slices_with_state_close_to_now(self, api, async_manager_mock, recent_start_date, some_config): """Stream will use start_date when close to now and start_date close to now""" start_date = recent_start_date end_date = pendulum.now() cursor_value = end_date - duration(days=1) state = {AdsInsights.cursor_field: cursor_value.date().isoformat()} - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) + stream = AdsInsights( + api=api, account_ids=some_config["account_ids"], start_date=start_date, end_date=end_date, insights_lookback_window=28 + ) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list(stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental)) - assert slices == [{"insight_job": 1}, {"insight_job": 2}, {"insight_job": 3}] + assert slices == [ + {"account_id": "unknown_account", "insight_job": 1}, + {"account_id": "unknown_account", "insight_job": 2}, + {"account_id": "unknown_account", "insight_job": 3}, + ] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) @@ -222,20 +330,36 @@ def test_stream_slices_with_state_close_to_now(self, api, async_manager_mock, re assert generated_jobs[0].interval.start == start_date.date() assert generated_jobs[1].interval.start == start_date.date() + duration(days=1) - def test_stream_slices_with_state_and_slices(self, api, async_manager_mock, start_date): + @pytest.mark.parametrize("state_format", ["old_format", "new_format"]) + def test_stream_slices_with_state_and_slices(self, api, async_manager_mock, start_date, some_config, state_format): """Stream will use cursor_value from state, but will skip saved slices""" end_date = start_date + duration(days=10) cursor_value = start_date + duration(days=5) - state = { - AdsInsights.cursor_field: cursor_value.date().isoformat(), - "slices": [(cursor_value + duration(days=1)).date().isoformat(), (cursor_value + duration(days=3)).date().isoformat()], - } - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) + + if state_format == "old_format": + state = { + AdsInsights.cursor_field: cursor_value.date().isoformat(), + "slices": [(cursor_value + duration(days=1)).date().isoformat(), (cursor_value + duration(days=3)).date().isoformat()], + } + else: + state = { + "unknown_account": { + AdsInsights.cursor_field: cursor_value.date().isoformat(), + "slices": [(cursor_value + duration(days=1)).date().isoformat(), (cursor_value + duration(days=3)).date().isoformat()], + } + } + stream = AdsInsights( + api=api, account_ids=some_config["account_ids"], start_date=start_date, end_date=end_date, insights_lookback_window=28 + ) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list(stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental)) - assert slices == [{"insight_job": 1}, {"insight_job": 2}, {"insight_job": 3}] + assert slices == [ + {"account_id": "unknown_account", "insight_job": 1}, + {"account_id": "unknown_account", "insight_job": 2}, + {"account_id": "unknown_account", "insight_job": 3}, + ] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) @@ -243,18 +367,25 @@ def test_stream_slices_with_state_and_slices(self, api, async_manager_mock, star assert generated_jobs[0].interval.start == cursor_value.date() + duration(days=2) assert generated_jobs[1].interval.start == cursor_value.date() + duration(days=4) - def test_get_json_schema(self, api): - stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) + def test_get_json_schema(self, api, some_config): + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=datetime(2010, 1, 1), + end_date=datetime(2011, 1, 1), + insights_lookback_window=28, + ) schema = stream.get_json_schema() assert "device_platform" not in schema["properties"] assert "country" not in schema["properties"] - assert not (set(stream.fields) - set(schema["properties"].keys())), "all fields present in schema" + assert not (set(stream.fields()) - set(schema["properties"].keys())), "all fields present in schema" - def test_get_json_schema_custom(self, api): + def test_get_json_schema_custom(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), breakdowns=["device_platform", "country"], @@ -265,38 +396,41 @@ def test_get_json_schema_custom(self, api): assert "device_platform" in schema["properties"] assert "country" in schema["properties"] - assert not (set(stream.fields) - set(schema["properties"].keys())), "all fields present in schema" + assert not (set(stream.fields()) - set(schema["properties"].keys())), "all fields present in schema" - def test_fields(self, api): + def test_fields(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28, ) - fields = stream.fields + fields = stream.fields() assert "account_id" in fields assert "account_currency" in fields assert "actions" in fields - def test_fields_custom(self, api): + def test_fields_custom(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), fields=["account_id", "account_currency"], insights_lookback_window=28, ) - assert stream.fields == ["account_id", "account_currency"] + assert stream.fields() == ["account_id", "account_currency"] schema = stream.get_json_schema() assert schema["properties"].keys() == set(["account_currency", "account_id", stream.cursor_field, "date_stop", "ad_id"]) - def test_level_custom(self, api): + def test_level_custom(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), fields=["account_id", "account_currency"], @@ -306,9 +440,10 @@ def test_level_custom(self, api): assert stream.level == "adset" - def test_breackdowns_fields_present_in_response_data(self, api): + def test_breackdowns_fields_present_in_response_data(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), breakdowns=["age", "gender"], diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py index 1f035c5c878e8..66604660645fd 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py @@ -9,7 +9,7 @@ from facebook_business import FacebookSession from facebook_business.api import FacebookAdsApi, FacebookAdsApiBatch from source_facebook_marketing.api import MyFacebookAdsApi -from source_facebook_marketing.streams.base_streams import FBMarketingStream +from source_facebook_marketing.streams.base_streams import FBMarketingIncrementalStream, FBMarketingStream @pytest.fixture(name="mock_batch_responses") @@ -96,3 +96,54 @@ def test_date_time_value(self): } }, } == record + + +class ConcreteFBMarketingIncrementalStream(FBMarketingIncrementalStream): + cursor_field = "date" + + def list_objects(self, **kwargs): + return [] + + +@pytest.fixture +def incremental_class_instance(api): + return ConcreteFBMarketingIncrementalStream(api=api, account_ids=["123", "456", "789"], start_date=None, end_date=None) + + +class TestFBMarketingIncrementalStreamSliceAndState: + def test_stream_slices_multiple_accounts_with_state(self, incremental_class_instance): + stream_state = {"123": {"state_key": "state_value"}, "456": {"state_key": "another_state_value"}} + expected_slices = [ + {"account_id": "123", "stream_state": {"state_key": "state_value"}}, + {"account_id": "456", "stream_state": {"state_key": "another_state_value"}}, + {"account_id": "789", "stream_state": {}}, + ] + assert list(incremental_class_instance.stream_slices(stream_state)) == expected_slices + + def test_stream_slices_multiple_accounts_empty_state(self, incremental_class_instance): + expected_slices = [ + {"account_id": "123", "stream_state": {}}, + {"account_id": "456", "stream_state": {}}, + {"account_id": "789", "stream_state": {}}, + ] + assert list(incremental_class_instance.stream_slices()) == expected_slices + + def test_stream_slices_single_account_with_state(self, incremental_class_instance): + incremental_class_instance._account_ids = ["123"] + stream_state = {"state_key": "state_value"} + expected_slices = [{"account_id": "123", "stream_state": stream_state}] + assert list(incremental_class_instance.stream_slices(stream_state)) == expected_slices + + def test_stream_slices_single_account_empty_state(self, incremental_class_instance): + incremental_class_instance._account_ids = ["123"] + expected_slices = [{"account_id": "123", "stream_state": None}] + assert list(incremental_class_instance.stream_slices()) == expected_slices + + def test_get_updated_state(self, incremental_class_instance): + current_stream_state = {"123": {"date": "2021-01-15T00:00:00+00:00"}, "include_deleted": False} + latest_record = {"account_id": "123", "date": "2021-01-20T00:00:00+00:00"} + + expected_state = {"123": {"date": "2021-01-20T00:00:00+00:00", "include_deleted": False}, "include_deleted": False} + + new_state = incremental_class_instance.get_updated_state(current_stream_state, latest_record) + assert new_state == expected_state diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py index 0f18516db1325..0d862aab6f319 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py @@ -52,7 +52,7 @@ def fb_call_amount_data_response_fixture(): class TestBackoff: - def test_limit_reached(self, mocker, requests_mock, api, fb_call_rate_response, account_id): + def test_limit_reached(self, mocker, requests_mock, api, fb_call_rate_response, account_id, some_config): """Error once, check that we retry and not fail""" # turn Campaigns into non batch mode to test non batch logic campaign_responses = [ @@ -67,9 +67,9 @@ def test_limit_reached(self, mocker, requests_mock, api, fb_call_rate_response, requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/1/", [{"status_code": 200}]) requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/2/", [{"status_code": 200}]) - stream = Campaigns(api=api, start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False) + stream = Campaigns(api=api, account_ids=[account_id], start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False) try: - records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) assert records except FacebookRequestError: pytest.fail("Call rate error has not being handled") @@ -111,12 +111,12 @@ def test_batch_limit_reached(self, requests_mock, api, fb_call_rate_response, ac requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/", responses) requests_mock.register_uri("POST", FacebookSession.GRAPH + f"/{FB_API_VERSION}/", batch_responses) - stream = AdCreatives(api=api, include_deleted=False) - records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + stream = AdCreatives(api=api, account_ids=[account_id], include_deleted=False) + records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) assert records == [ - {"id": "123", "object_type": "SHARE", "status": "ACTIVE"}, - {"id": "1234", "object_type": "SHARE", "status": "ACTIVE"}, + {"account_id": "unknown_account", "id": "123", "object_type": "SHARE", "status": "ACTIVE"}, + {"account_id": "unknown_account", "id": "1234", "object_type": "SHARE", "status": "ACTIVE"}, ] @pytest.mark.parametrize( @@ -130,7 +130,7 @@ def test_batch_limit_reached(self, requests_mock, api, fb_call_rate_response, ac ) def test_common_error_retry(self, error_response, requests_mock, api, account_id): """Error once, check that we retry and not fail""" - account_data = {"id": 1, "updated_time": "2020-09-25T00:00:00Z", "name": "Some name"} + account_data = {"account_id": "unknown_account", "id": 1, "updated_time": "2020-09-25T00:00:00Z", "name": "Some name"} responses = [ error_response, { @@ -143,8 +143,8 @@ def test_common_error_retry(self, error_response, requests_mock, api, account_id requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/", responses) requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{account_data['id']}/", responses) - stream = AdAccount(api=api) - accounts = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + stream = AdAccount(api=api, account_ids=[account_id]) + accounts = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) assert accounts == [account_data] @@ -155,9 +155,11 @@ def test_limit_error_retry(self, fb_call_amount_data_response, requests_mock, ap "GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/campaigns", [fb_call_amount_data_response] ) - stream = Campaigns(api=api, start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100) + stream = Campaigns( + api=api, account_ids=[account_id], start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100 + ) try: - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) except AirbyteTracedException: assert [x.qs.get("limit")[0] for x in res.request_history] == ["100", "50", "25", "12", "6"] @@ -192,9 +194,11 @@ def test_limit_error_retry_revert_page_size(self, requests_mock, api, account_id [error, success, error, success], ) - stream = Activities(api=api, start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100) + stream = Activities( + api=api, account_ids=[account_id], start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100 + ) try: - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) except FacebookRequestError: assert [x.qs.get("limit")[0] for x in res.request_history] == ["100", "50", "100", "50"] @@ -218,8 +222,8 @@ def test_start_date_not_provided(self, requests_mock, api, account_id): [success], ) - stream = Activities(api=api, start_date=None, end_date=None, include_deleted=False, page_size=100) - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + stream = Activities(api=api, account_ids=[account_id], start_date=None, end_date=None, include_deleted=False, page_size=100) + list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) def test_limit_error_retry_next_page(self, fb_call_amount_data_response, requests_mock, api, account_id): """Unlike the previous test, this one tests the API call fail on the second or more page of a request.""" @@ -240,8 +244,10 @@ def test_limit_error_retry_next_page(self, fb_call_amount_data_response, request ], ) - stream = Videos(api=api, start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100) + stream = Videos( + api=api, account_ids=[account_id], start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100 + ) try: - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) except AirbyteTracedException: assert [x.qs.get("limit")[0] for x in res.request_history] == ["100", "100", "50", "25", "12", "6"] diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py new file mode 100644 index 0000000000000..092b855396c13 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py @@ -0,0 +1,87 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json +from typing import Any, Mapping + +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source +from source_facebook_marketing.config_migrations import MigrateAccountIdToArray +from source_facebook_marketing.source import SourceFacebookMarketing + +# BASE ARGS +CMD = "check" +TEST_CONFIG_PATH = "unit_tests/test_migrations/test_old_config.json" +NEW_TEST_CONFIG_PATH = "unit_tests/test_migrations/test_new_config.json" +UPGRADED_TEST_CONFIG_PATH = "unit_tests/test_migrations/test_upgraded_config.json" +SOURCE_INPUT_ARGS = [CMD, "--config", TEST_CONFIG_PATH] +SOURCE: Source = SourceFacebookMarketing() + + +# HELPERS +def load_config(config_path: str = TEST_CONFIG_PATH) -> Mapping[str, Any]: + with open(config_path, "r") as config: + return json.load(config) + + +def revert_migration(config_path: str = TEST_CONFIG_PATH) -> None: + with open(config_path, "r") as test_config: + config = json.load(test_config) + config.pop("account_ids") + with open(config_path, "w") as updated_config: + config = json.dumps(config) + updated_config.write(config) + + +def test_migrate_config(): + migration_instance = MigrateAccountIdToArray() + original_config = load_config() + # migrate the test_config + migration_instance.migrate(SOURCE_INPUT_ARGS, SOURCE) + # load the updated config + test_migrated_config = load_config() + # check migrated property + assert "account_ids" in test_migrated_config + assert isinstance(test_migrated_config["account_ids"], list) + # check the old property is in place + assert "account_id" in test_migrated_config + assert isinstance(test_migrated_config["account_id"], str) + # check the migration should be skipped, once already done + assert not migration_instance.should_migrate(test_migrated_config) + # load the old custom reports VS migrated + assert [original_config["account_id"]] == test_migrated_config["account_ids"] + # test CONTROL MESSAGE was emitted + control_msg = migration_instance.message_repository._message_queue[0] + assert control_msg.type == Type.CONTROL + assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG + # old custom_reports are stil type(str) + assert isinstance(control_msg.control.connectorConfig.config["account_id"], str) + # new custom_reports are type(list) + assert isinstance(control_msg.control.connectorConfig.config["account_ids"], list) + # check the migrated values + assert control_msg.control.connectorConfig.config["account_ids"] == ["01234567890"] + # revert the test_config to the starting point + revert_migration() + + +def test_config_is_reverted(): + # check the test_config state, it has to be the same as before tests + test_config = load_config() + # check the config no longer has the migarted property + assert "account_ids" not in test_config + # check the old property is still there + assert "account_id" in test_config + assert isinstance(test_config["account_id"], str) + + +def test_should_not_migrate_new_config(): + new_config = load_config(NEW_TEST_CONFIG_PATH) + migration_instance = MigrateAccountIdToArray() + assert not migration_instance.should_migrate(new_config) + +def test_should_not_migrate_upgraded_config(): + new_config = load_config(UPGRADED_TEST_CONFIG_PATH) + migration_instance = MigrateAccountIdToArray() + assert not migration_instance.should_migrate(new_config) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py index 372ca7c5cdd2e..105306b25f555 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py @@ -15,7 +15,7 @@ FB_API_VERSION = FacebookAdsApi.API_VERSION account_id = "unknown_account" -some_config = {"start_date": "2021-01-23T00:00:00Z", "account_id": account_id, "access_token": "unknown_token"} +some_config = {"start_date": "2021-01-23T00:00:00Z", "account_ids": [account_id], "access_token": "unknown_token"} base_url = f"{FacebookSession.GRAPH}/{FB_API_VERSION}/" act_url = f"{base_url}act_{account_id}/" @@ -26,8 +26,8 @@ } } ad_creative_data = [ - {"id": "111111", "name": "ad creative 1", "updated_time": "2023-03-21T22:33:56-0700"}, - {"id": "222222", "name": "ad creative 2", "updated_time": "2023-03-22T22:33:56-0700"}, + {"account_id": account_id, "id": "111111", "name": "ad creative 1", "updated_time": "2023-03-21T22:33:56-0700"}, + {"account_id": account_id, "id": "222222", "name": "ad creative 2", "updated_time": "2023-03-22T22:33:56-0700"}, ] ad_creative_response = { "json": { @@ -288,9 +288,11 @@ def test_retryable_error(self, some_config, requests_mock, name, retryable_error requests_mock.register_uri("GET", f"{act_url}", [retryable_error_response, ad_account_response]) requests_mock.register_uri("GET", f"{act_url}adcreatives", [retryable_error_response, ad_creative_response]) - api = API(account_id=some_config["account_id"], access_token=some_config["access_token"], page_size=100) - stream = AdCreatives(api=api, include_deleted=False) - ad_creative_records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + api = API(access_token=some_config["access_token"], page_size=100) + stream = AdCreatives(api=api, account_ids=some_config["account_ids"], include_deleted=False) + ad_creative_records = list( + stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id}) + ) assert ad_creative_records == ad_creative_data @@ -301,12 +303,12 @@ def test_retryable_error(self, some_config, requests_mock, name, retryable_error def test_config_error_during_account_info_read(self, requests_mock, name, friendly_msg, config_error_response): """Error raised during account info read""" - api = API(account_id=some_config["account_id"], access_token=some_config["access_token"], page_size=100) - stream = AdCreatives(api=api, include_deleted=False) + api = API(access_token=some_config["access_token"], page_size=100) + stream = AdCreatives(api=api, account_ids=some_config["account_ids"], include_deleted=False) requests_mock.register_uri("GET", f"{act_url}", [config_error_response, ad_account_response]) try: - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) assert False except Exception as error: assert isinstance(error, AirbyteTracedException) @@ -318,13 +320,13 @@ def test_config_error_during_account_info_read(self, requests_mock, name, friend def test_config_error_during_actual_nodes_read(self, requests_mock, name, friendly_msg, config_error_response): """Error raised during actual nodes read""" - api = API(account_id=some_config["account_id"], access_token=some_config["access_token"], page_size=100) - stream = AdCreatives(api=api, include_deleted=False) + api = API(access_token=some_config["access_token"], page_size=100) + stream = AdCreatives(api=api, account_ids=some_config["account_ids"], include_deleted=False) requests_mock.register_uri("GET", f"{act_url}", [ad_account_response]) requests_mock.register_uri("GET", f"{act_url}adcreatives", [config_error_response, ad_creative_response]) try: - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) assert False except Exception as error: assert isinstance(error, AirbyteTracedException) @@ -335,9 +337,10 @@ def test_config_error_during_actual_nodes_read(self, requests_mock, name, friend def test_config_error_insights_account_info_read(self, requests_mock, name, friendly_msg, config_error_response): """Error raised during actual nodes read""" - api = API(account_id=some_config["account_id"], access_token=some_config["access_token"], page_size=100) + api = API(access_token=some_config["access_token"], page_size=100) stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), fields=["account_id", "account_currency"], @@ -357,9 +360,10 @@ def test_config_error_insights_account_info_read(self, requests_mock, name, frie def test_config_error_insights_during_actual_nodes_read(self, requests_mock, name, friendly_msg, config_error_response): """Error raised during actual nodes read""" - api = API(account_id=some_config["account_id"], access_token=some_config["access_token"], page_size=100) + api = API(access_token=some_config["access_token"], page_size=100) stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), fields=["account_id", "account_currency"], @@ -411,8 +415,11 @@ def test_adaccount_list_objects_retry(self, requests_mock, failure_response): ] As a workaround for this case we can retry the API call excluding `owner` from `?fields=` GET query param. """ - api = API(account_id=some_config["account_id"], access_token=some_config["access_token"], page_size=100) - stream = AdAccount(api=api) + api = API(access_token=some_config["access_token"], page_size=100) + stream = AdAccount( + api=api, + account_ids=some_config["account_ids"], + ) business_user = {"account_id": account_id, "business": {"id": "1", "name": "TEST"}} requests_mock.register_uri("GET", f"{base_url}me/business_users", status_code=200, json=business_user) @@ -423,5 +430,5 @@ def test_adaccount_list_objects_retry(self, requests_mock, failure_response): success_response = {"status_code": 200, "json": {"account_id": account_id}} requests_mock.register_uri("GET", f"{act_url}", [failure_response, success_response]) - record_gen = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=None, stream_state={}) + record_gen = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"account_id": account_id}, stream_state={}) assert list(record_gen) == [{"account_id": "unknown_account", "id": "act_unknown_account"}] diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_new_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_new_config.json new file mode 100644 index 0000000000000..489ff3fd68fb2 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_new_config.json @@ -0,0 +1,14 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_ids": ["01234567890"], + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_old_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_old_config.json new file mode 100644 index 0000000000000..a04560eb77103 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_old_config.json @@ -0,0 +1,14 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_id": "01234567890", + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_upgraded_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_upgraded_config.json new file mode 100644 index 0000000000000..648b4e2c390b1 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_upgraded_config.json @@ -0,0 +1,15 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_id": "01234567890", + "account_ids": ["01234567890"], + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py index 98bb41ad72f94..7b96c5ced3b9c 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py @@ -4,6 +4,7 @@ from copy import deepcopy +from unittest.mock import call import pytest from airbyte_cdk.models import ( @@ -26,7 +27,7 @@ @pytest.fixture(name="config") def config_fixture(requests_mock): config = { - "account_id": "123", + "account_ids": ["123"], "access_token": "TOKEN", "start_date": "2019-10-10T00:00:00Z", "end_date": "2020-10-10T00:00:00Z", @@ -50,7 +51,7 @@ def inner(**kwargs): @pytest.fixture(name="api") def api_fixture(mocker): api_mock = mocker.patch("source_facebook_marketing.source.API") - api_mock.return_value = mocker.Mock(account=123) + api_mock.return_value = mocker.Mock(account=mocker.Mock(return_value=123)) return api_mock @@ -82,8 +83,13 @@ def test_check_connection_find_account_was_called(self, api_find_account, config """Check if _find_account was called to validate credentials""" ok, error_msg = fb_marketing.check_connection(logger_mock, config=config) - api_find_account.assert_called_once_with(config["account_id"]) - logger_mock.info.assert_called_once_with("Select account 1234") + api_find_account.assert_called_once_with(config["account_ids"][0]) + logger_mock.info.assert_has_calls( + [ + call("Attempting to retrieve information for account with ID: 123"), + call("Successfully retrieved account information for account: 1234"), + ] + ) assert ok assert not error_msg diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py index 12f493ce37e5c..a2b03c52e67ca 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py @@ -19,7 +19,7 @@ from source_facebook_marketing.streams.streams import fetch_thumbnail_data_url -def test_filter_all_statuses(api, mocker): +def test_filter_all_statuses(api, mocker, some_config): mocker.patch.multiple(FBMarketingStream, __abstractmethods__=set()) expected = { "filtering": [ @@ -45,7 +45,7 @@ def test_filter_all_statuses(api, mocker): } ] } - assert FBMarketingStream(api=api)._filter_all_statuses() == expected + assert FBMarketingStream(api=api, account_ids=some_config["account_ids"])._filter_all_statuses() == expected @pytest.mark.parametrize( @@ -76,15 +76,27 @@ def test_parse_call_rate_header(): [AdsInsightsRegion, ["region"], ["action_type", "action_target_id", "action_destination"]], ], ) -def test_ads_insights_breakdowns(class_name, breakdowns, action_breakdowns): - kwargs = {"api": None, "start_date": pendulum.now(), "end_date": pendulum.now(), "insights_lookback_window": 1} +def test_ads_insights_breakdowns(class_name, breakdowns, action_breakdowns, some_config): + kwargs = { + "api": None, + "account_ids": some_config["account_ids"], + "start_date": pendulum.now(), + "end_date": pendulum.now(), + "insights_lookback_window": 1, + } stream = class_name(**kwargs) assert stream.breakdowns == breakdowns assert stream.action_breakdowns == action_breakdowns -def test_custom_ads_insights_breakdowns(): - kwargs = {"api": None, "start_date": pendulum.now(), "end_date": pendulum.now(), "insights_lookback_window": 1} +def test_custom_ads_insights_breakdowns(some_config): + kwargs = { + "api": None, + "account_ids": some_config["account_ids"], + "start_date": pendulum.now(), + "end_date": pendulum.now(), + "insights_lookback_window": 1, + } stream = AdsInsights(breakdowns=["mmm"], action_breakdowns=["action_destination"], **kwargs) assert stream.breakdowns == ["mmm"] assert stream.action_breakdowns == ["action_destination"] @@ -98,9 +110,10 @@ def test_custom_ads_insights_breakdowns(): assert stream.action_breakdowns == [] -def test_custom_ads_insights_action_report_times(): +def test_custom_ads_insights_action_report_times(some_config): kwargs = { "api": None, + "account_ids": some_config["account_ids"], "start_date": pendulum.now(), "end_date": pendulum.now(), "insights_lookback_window": 1, diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index fb39df1d2d82c..75eff14ff67cd 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -70,7 +70,7 @@ You can use the [Access Token Tool](https://developers.facebook.com/tools/access #### Facebook Marketing Source Settings -1. For **Account ID**, enter the [Facebook Ad Account ID Number](https://www.facebook.com/business/help/1492627900875762) to use when pulling data from the Facebook Marketing API. To find this ID, open your Meta Ads Manager. The Ad Account ID number is in the **Account** dropdown menu or in your browser's address bar. Refer to the [Facebook docs](https://www.facebook.com/business/help/1492627900875762) for more information. +1. For **Account ID(s)**, enter one or multiple comma-separated [Facebook Ad Account ID Numbers](https://www.facebook.com/business/help/1492627900875762) to use when pulling data from the Facebook Marketing API. To find this ID, open your Meta Ads Manager. The Ad Account ID number is in the **Account** dropdown menu or in your browser's address bar. Refer to the [Facebook docs](https://www.facebook.com/business/help/1492627900875762) for more information. 2. (Optional) For **Start Date**, use the provided datepicker, or enter the date programmatically in the `YYYY-MM-DDTHH:mm:ssZ` format. If not set then all data will be replicated for usual streams and only last 2 years for insight streams. :::warning @@ -201,133 +201,134 @@ The Facebook Marketing connector uses the `lookback_window` parameter to repeate ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 1.2.3 | 2024-01-04 | [33934](https://github.com/airbytehq/airbyte/pull/33828) | Make ready for airbyte-lib | -| 1.2.2 | 2024-01-02 | [33828](https://github.com/airbytehq/airbyte/pull/33828) | Add insights job timeout to be an option, so a user can specify their own value | -| 1.2.1 | 2023-11-22 | [32731](https://github.com/airbytehq/airbyte/pull/32731) | Removed validation that blocked personal ad accounts during `check` | -| 1.2.0 | 2023-10-31 | [31999](https://github.com/airbytehq/airbyte/pull/31999) | Extend the `AdCreatives` stream schema | -| 1.1.17 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 1.1.16 | 2023-10-11 | [31284](https://github.com/airbytehq/airbyte/pull/31284) | Fix error occurring when trying to access the `funding_source_details` field of the `AdAccount` stream | -| 1.1.15 | 2023-10-06 | [31132](https://github.com/airbytehq/airbyte/pull/31132) | Fix permission error for `AdAccount` stream | -| 1.1.14 | 2023-09-26 | [30758](https://github.com/airbytehq/airbyte/pull/30758) | Exception should not be raises if a stream is not found | -| 1.1.13 | 2023-09-22 | [30706](https://github.com/airbytehq/airbyte/pull/30706) | Performance testing - include socat binary in docker image | -| 1.1.12 | 2023-09-22 | [30655](https://github.com/airbytehq/airbyte/pull/30655) | Updated doc; improved schema for custom insight streams; updated SAT or custom insight streams; removed obsolete optional max_batch_size option from spec | -| 1.1.11 | 2023-09-21 | [30650](https://github.com/airbytehq/airbyte/pull/30650) | Fix None issue since start_date is optional | -| 1.1.10 | 2023-09-15 | [30485](https://github.com/airbytehq/airbyte/pull/30485) | added 'status' and 'configured_status' fields for campaigns stream schema | -| 1.1.9 | 2023-08-31 | [29994](https://github.com/airbytehq/airbyte/pull/29994) | Removed batch processing, updated description in specs, added user-friendly error message, removed start_date from required attributes | -| 1.1.8 | 2023-09-04 | [29666](https://github.com/airbytehq/airbyte/pull/29666) | Adding custom field `boosted_object_id` to a streams schema in `campaigns` catalog `CustomAudiences` | -| 1.1.7 | 2023-08-21 | [29674](https://github.com/airbytehq/airbyte/pull/29674) | Exclude `rule` from stream `CustomAudiences` | -| 1.1.6 | 2023-08-18 | [29642](https://github.com/airbytehq/airbyte/pull/29642) | Stop batch requests if only 1 left in a batch | -| 1.1.5 | 2023-08-18 | [29610](https://github.com/airbytehq/airbyte/pull/29610) | Automatically reduce batch size | -| 1.1.4 | 2023-08-08 | [29412](https://github.com/airbytehq/airbyte/pull/29412) | Add new custom_audience stream | -| 1.1.3 | 2023-08-08 | [29208](https://github.com/airbytehq/airbyte/pull/29208) | Add account type validation during check | -| 1.1.2 | 2023-08-03 | [29042](https://github.com/airbytehq/airbyte/pull/29042) | Fix broken `advancedAuth` references for `spec` | -| 1.1.1 | 2023-07-26 | [27996](https://github.com/airbytehq/airbyte/pull/27996) | Remove reference to authSpecification | -| 1.1.0 | 2023-07-11 | [26345](https://github.com/airbytehq/airbyte/pull/26345) | Add new `action_report_time` attribute to `AdInsights` class | -| 1.0.1 | 2023-07-07 | [27979](https://github.com/airbytehq/airbyte/pull/27979) | Added the ability to restore the reduced request record limit after the successful retry, and handle the `unknown error` (code 99) with the retry strategy | -| 1.0.0 | 2023-07-05 | [27563](https://github.com/airbytehq/airbyte/pull/27563) | Migrate to FB SDK version 17 | -| 0.5.0 | 2023-06-26 | [27728](https://github.com/airbytehq/airbyte/pull/27728) | License Update: Elv2 | -| 0.4.3 | 2023-05-12 | [27483](https://github.com/airbytehq/airbyte/pull/27483) | Reduce replication start date by one more day | -| 0.4.2 | 2023-06-09 | [27201](https://github.com/airbytehq/airbyte/pull/27201) | Add `complete_oauth_server_output_specification` to spec | -| 0.4.1 | 2023-06-02 | [26941](https://github.com/airbytehq/airbyte/pull/26941) | Remove `authSpecification` from spec.json, use `advanced_auth` instead | -| 0.4.0 | 2023-05-29 | [26720](https://github.com/airbytehq/airbyte/pull/26720) | Add Prebuilt Ad Insights reports | -| 0.3.7 | 2023-05-12 | [26000](https://github.com/airbytehq/airbyte/pull/26000) | Handle config errors | -| 0.3.6 | 2023-04-27 | [22999](https://github.com/airbytehq/airbyte/pull/22999) | Specified date formatting in specification | -| 0.3.5 | 2023-04-26 | [24994](https://github.com/airbytehq/airbyte/pull/24994) | Emit stream status messages | -| 0.3.4 | 2023-04-18 | [22990](https://github.com/airbytehq/airbyte/pull/22990) | Increase pause interval | -| 0.3.3 | 2023-04-14 | [25204](https://github.com/airbytehq/airbyte/pull/25204) | Fix data retention period validation | -| 0.3.2 | 2023-04-08 | [25003](https://github.com/airbytehq/airbyte/pull/25003) | Don't fetch `thumbnail_data_url` if it's None | -| 0.3.1 | 2023-03-27 | [24600](https://github.com/airbytehq/airbyte/pull/24600) | Reduce request record limit when retrying second page or further | -| 0.3.0 | 2023-03-16 | [19141](https://github.com/airbytehq/airbyte/pull/19141) | Added Level parameter to custom Ads Insights | -| 0.2.86 | 2023-03-01 | [23625](https://github.com/airbytehq/airbyte/pull/23625) | Add user friendly fields description in spec and docs. Extend error message for invalid Account ID case. | -| 0.2.85 | 2023-02-14 | [23003](https://github.com/airbytehq/airbyte/pull/23003) | Bump facebook_business to 16.0.0 | -| 0.2.84 | 2023-01-27 | [22003](https://github.com/airbytehq/airbyte/pull/22003) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.2.83 | 2023-01-13 | [21149](https://github.com/airbytehq/airbyte/pull/21149) | Videos stream remove filtering | -| 0.2.82 | 2023-01-09 | [21149](https://github.com/airbytehq/airbyte/pull/21149) | Fix AdAccount schema | -| 0.2.81 | 2023-01-05 | [21057](https://github.com/airbytehq/airbyte/pull/21057) | Remove unsupported fields from request | -| 0.2.80 | 2022-12-21 | [20736](https://github.com/airbytehq/airbyte/pull/20736) | Fix update next cursor | -| 0.2.79 | 2022-12-07 | [20402](https://github.com/airbytehq/airbyte/pull/20402) | Exclude Not supported fields from request | -| 0.2.78 | 2022-12-07 | [20165](https://github.com/airbytehq/airbyte/pull/20165) | Fix fields permission error | -| 0.2.77 | 2022-12-06 | [20131](https://github.com/airbytehq/airbyte/pull/20131) | Update next cursor value at read start | -| 0.2.76 | 2022-12-03 | [20043](https://github.com/airbytehq/airbyte/pull/20043) | Allows `action_breakdowns` to be an empty list - bugfix for #20016 | -| 0.2.75 | 2022-12-03 | [20016](https://github.com/airbytehq/airbyte/pull/20016) | Allows `action_breakdowns` to be an empty list | -| 0.2.74 | 2022-11-25 | [19803](https://github.com/airbytehq/airbyte/pull/19803) | New default for `action_breakdowns`, improve "check" command speed | -| 0.2.73 | 2022-11-21 | [19645](https://github.com/airbytehq/airbyte/pull/19645) | Check "breakdowns" combinations | -| 0.2.72 | 2022-11-04 | [18971](https://github.com/airbytehq/airbyte/pull/18971) | Handle FacebookBadObjectError for empty results on async jobs | -| 0.2.71 | 2022-10-31 | [18734](https://github.com/airbytehq/airbyte/pull/18734) | Reduce request record limit on retry | -| 0.2.70 | 2022-10-26 | [18045](https://github.com/airbytehq/airbyte/pull/18045) | Upgrade FB SDK to v15.0 | -| 0.2.69 | 2022-10-17 | [18045](https://github.com/airbytehq/airbyte/pull/18045) | Remove "pixel" field from the Custom Conversions stream schema | -| 0.2.68 | 2022-10-12 | [17869](https://github.com/airbytehq/airbyte/pull/17869) | Remove "format" from optional datetime `end_date` field | -| 0.2.67 | 2022-10-04 | [17551](https://github.com/airbytehq/airbyte/pull/17551) | Add `cursor_field` for custom_insights stream schema | -| 0.2.65 | 2022-09-29 | [17371](https://github.com/airbytehq/airbyte/pull/17371) | Fix stream CustomConversions `enable_deleted=False` | -| 0.2.64 | 2022-09-22 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | -| 0.2.64 | 2022-09-22 | [17027](https://github.com/airbytehq/airbyte/pull/17027) | Limit time range with 37 months when creating an insight job from lower edge object. Retry bulk request when getting error code `960` | -| 0.2.63 | 2022-09-06 | [15724](https://github.com/airbytehq/airbyte/pull/15724) | Add the Custom Conversion stream | -| 0.2.62 | 2022-09-01 | [16222](https://github.com/airbytehq/airbyte/pull/16222) | Remove `end_date` from config if empty value (re-implement #16096) | -| 0.2.61 | 2022-08-29 | [16096](https://github.com/airbytehq/airbyte/pull/16096) | Remove `end_date` from config if empty value | -| 0.2.60 | 2022-08-19 | [15788](https://github.com/airbytehq/airbyte/pull/15788) | Retry FacebookBadObjectError | -| 0.2.59 | 2022-08-04 | [15327](https://github.com/airbytehq/airbyte/pull/15327) | Shift date validation from config validation to stream method | -| 0.2.58 | 2022-07-25 | [15012](https://github.com/airbytehq/airbyte/pull/15012) | Add `DATA_RETENTION_PERIOD`validation and fix `failed_delivery_checks` field schema type issue | -| 0.2.57 | 2022-07-25 | [14831](https://github.com/airbytehq/airbyte/pull/14831) | Update Facebook SDK to version 14.0.0 | -| 0.2.56 | 2022-07-19 | [14831](https://github.com/airbytehq/airbyte/pull/14831) | Add future `start_date` and `end_date` validation | -| 0.2.55 | 2022-07-18 | [14786](https://github.com/airbytehq/airbyte/pull/14786) | Check if the authorized user has the "MANAGE" task permission when getting the `funding_source_details` field in the ad_account stream | -| 0.2.54 | 2022-06-29 | [14267](https://github.com/airbytehq/airbyte/pull/14267) | Make MAX_BATCH_SIZE available in config | -| 0.2.53 | 2022-06-16 | [13623](https://github.com/airbytehq/airbyte/pull/13623) | Add fields `bid_amount` `bid_strategy` `bid_constraints` to `ads_set` stream | -| 0.2.52 | 2022-06-14 | [13749](https://github.com/airbytehq/airbyte/pull/13749) | Fix the `not syncing any data` issue | -| 0.2.51 | 2022-05-30 | [13317](https://github.com/airbytehq/airbyte/pull/13317) | Change tax_id to string (Canadian has letter in tax_id) | -| 0.2.50 | 2022-04-27 | [12402](https://github.com/airbytehq/airbyte/pull/12402) | Add lookback window to insights streams | -| 0.2.49 | 2022-05-20 | [13047](https://github.com/airbytehq/airbyte/pull/13047) | Fix duplicating records during insights lookback period | -| 0.2.48 | 2022-05-19 | [13008](https://github.com/airbytehq/airbyte/pull/13008) | Update CDK to v0.1.58 avoid crashing on incorrect stream schemas | -| 0.2.47 | 2022-05-06 | [12685](https://github.com/airbytehq/airbyte/pull/12685) | Update CDK to v0.1.56 to emit an `AirbyeTraceMessage` on uncaught exceptions | -| 0.2.46 | 2022-04-22 | [12171](https://github.com/airbytehq/airbyte/pull/12171) | Allow configuration of page_size for requests | -| 0.2.45 | 2022-05-03 | [12390](https://github.com/airbytehq/airbyte/pull/12390) | Better retry logic for split-up async jobs | -| 0.2.44 | 2022-04-14 | [11751](https://github.com/airbytehq/airbyte/pull/11751) | Update API to a directly initialise an AdAccount with the given ID | -| 0.2.43 | 2022-04-13 | [11801](https://github.com/airbytehq/airbyte/pull/11801) | Fix `user_tos_accepted` schema to be an object | -| 0.2.42 | 2022-04-06 | [11761](https://github.com/airbytehq/airbyte/pull/11761) | Upgrade Facebook Python SDK to version 13 | -| 0.2.41 | 2022-03-28 | [11446](https://github.com/airbytehq/airbyte/pull/11446) | Increase number of attempts for individual jobs | -| 0.2.40 | 2022-02-28 | [10698](https://github.com/airbytehq/airbyte/pull/10698) | Improve sleeps time in rate limit handler | -| 0.2.39 | 2022-03-09 | [10917](https://github.com/airbytehq/airbyte/pull/10917) | Retry connections when FB API returns error code 2 (temporary oauth error) | -| 0.2.38 | 2022-03-08 | [10531](https://github.com/airbytehq/airbyte/pull/10531) | Add `time_increment` parameter to custom insights | -| 0.2.37 | 2022-02-28 | [10655](https://github.com/airbytehq/airbyte/pull/10655) | Add Activities stream | -| 0.2.36 | 2022-02-24 | [10588](https://github.com/airbytehq/airbyte/pull/10588) | Fix `execute_in_batch` for large amount of requests | -| 0.2.35 | 2022-02-18 | [10348](https://github.com/airbytehq/airbyte/pull/10348) | Add error code 104 to backoff triggers | -| 0.2.34 | 2022-02-17 | [10180](https://github.com/airbytehq/airbyte/pull/9805) | Performance and reliability fixes | -| 0.2.33 | 2021-12-28 | [10180](https://github.com/airbytehq/airbyte/pull/10180) | Add AdAccount and Images streams | -| 0.2.32 | 2022-01-07 | [10138](https://github.com/airbytehq/airbyte/pull/10138) | Add `primary_key` for all insights streams. | -| 0.2.31 | 2021-12-29 | [9138](https://github.com/airbytehq/airbyte/pull/9138) | Fix videos stream format field incorrect type | -| 0.2.30 | 2021-12-20 | [8962](https://github.com/airbytehq/airbyte/pull/8962) | Add `asset_feed_spec` field to `ad creatives` stream | -| 0.2.29 | 2021-12-17 | [8649](https://github.com/airbytehq/airbyte/pull/8649) | Retrieve ad_creatives image as data encoded | -| 0.2.28 | 2021-12-13 | [8742](https://github.com/airbytehq/airbyte/pull/8742) | Fix for schema generation related to "breakdown" fields | -| 0.2.27 | 2021-11-29 | [8257](https://github.com/airbytehq/airbyte/pull/8257) | Add fields to Campaign stream | -| 0.2.26 | 2021-11-19 | [7855](https://github.com/airbytehq/airbyte/pull/7855) | Add Video stream | -| 0.2.25 | 2021-11-12 | [7904](https://github.com/airbytehq/airbyte/pull/7904) | Implement retry logic for async jobs | -| 0.2.24 | 2021-11-09 | [7744](https://github.com/airbytehq/airbyte/pull/7744) | Fix fail when async job takes too long | -| 0.2.23 | 2021-11-08 | [7734](https://github.com/airbytehq/airbyte/pull/7734) | Resolve $ref field for discover schema | -| 0.2.22 | 2021-11-05 | [7605](https://github.com/airbytehq/airbyte/pull/7605) | Add job retry logics to AdsInsights stream | -| 0.2.21 | 2021-10-05 | [4864](https://github.com/airbytehq/airbyte/pull/4864) | Update insights streams with custom entries for fields, breakdowns and action_breakdowns | -| 0.2.20 | 2021-10-04 | [6719](https://github.com/airbytehq/airbyte/pull/6719) | Update version of facebook_business package to 12.0 | -| 0.2.19 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | -| 0.2.18 | 2021-09-28 | [6499](https://github.com/airbytehq/airbyte/pull/6499) | Fix field values converting fail | -| 0.2.17 | 2021-09-14 | [4978](https://github.com/airbytehq/airbyte/pull/4978) | Convert values' types according to schema types | -| 0.2.16 | 2021-09-14 | [6060](https://github.com/airbytehq/airbyte/pull/6060) | Fix schema for `ads_insights` stream | -| 0.2.15 | 2021-09-14 | [5958](https://github.com/airbytehq/airbyte/pull/5958) | Fix url parsing and add report that exposes conversions | -| 0.2.14 | 2021-07-19 | [4820](https://github.com/airbytehq/airbyte/pull/4820) | Improve the rate limit management | -| 0.2.12 | 2021-06-20 | [3743](https://github.com/airbytehq/airbyte/pull/3743) | Refactor connector to use CDK: - Improve error handling. - Improve async job performance \(insights\). - Add new configuration parameter `insights_days_per_job`. - Rename stream `adsets` to `ad_sets`. - Refactor schema logic for insights, allowing to configure any possible insight stream. | -| 0.2.10 | 2021-06-16 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Update version of facebook_business to 11.0 | -| 0.2.9 | 2021-06-10 | [3996](https://github.com/airbytehq/airbyte/pull/3996) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | -| 0.2.8 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add 80000 as a rate-limiting error code | -| 0.2.7 | 2021-06-03 | [3646](https://github.com/airbytehq/airbyte/pull/3646) | Add missing fields to AdInsights streams | -| 0.2.6 | 2021-05-25 | [3525](https://github.com/airbytehq/airbyte/pull/3525) | Fix handling call rate limit | -| 0.2.5 | 2021-05-20 | [3396](https://github.com/airbytehq/airbyte/pull/3396) | Allow configuring insights lookback window | -| 0.2.4 | 2021-05-13 | [3395](https://github.com/airbytehq/airbyte/pull/3395) | Fix an issue that caused losing Insights data from the past 28 days while incremental sync | -| 0.2.3 | 2021-04-28 | [3116](https://github.com/airbytehq/airbyte/pull/3116) | Wait longer \(5 min\) for async jobs to start | -| 0.2.2 | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726) | Fix base connector versioning | -| 0.2.1 | 2021-03-12 | [2391](https://github.com/airbytehq/airbyte/pull/2391) | Support FB Marketing API v10 | -| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | -| 0.1.4 | 2021-02-24 | [1902](https://github.com/airbytehq/airbyte/pull/1902) | Add `include_deleted` option in params | -| 0.1.3 | 2021-02-15 | [1990](https://github.com/airbytehq/airbyte/pull/1990) | Support Insights stream via async queries | -| 0.1.2 | 2021-01-22 | [1699](https://github.com/airbytehq/airbyte/pull/1699) | Add incremental support | -| 0.1.1 | 2021-01-15 | [1552](https://github.com/airbytehq/airbyte/pull/1552) | Release Native Facebook Marketing Connector | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.3.0 | 2024-01-09 | [33538](https://github.com/airbytehq/airbyte/pull/33538) | Updated the `Ad Account ID(s)` property to support multiple IDs | +| 1.2.3 | 2024-01-04 | [33934](https://github.com/airbytehq/airbyte/pull/33828) | Make ready for airbyte-lib | +| 1.2.2 | 2024-01-02 | [33828](https://github.com/airbytehq/airbyte/pull/33828) | Add insights job timeout to be an option, so a user can specify their own value | +| 1.2.1 | 2023-11-22 | [32731](https://github.com/airbytehq/airbyte/pull/32731) | Removed validation that blocked personal ad accounts during `check` | +| 1.2.0 | 2023-10-31 | [31999](https://github.com/airbytehq/airbyte/pull/31999) | Extend the `AdCreatives` stream schema | +| 1.1.17 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.1.16 | 2023-10-11 | [31284](https://github.com/airbytehq/airbyte/pull/31284) | Fix error occurring when trying to access the `funding_source_details` field of the `AdAccount` stream | +| 1.1.15 | 2023-10-06 | [31132](https://github.com/airbytehq/airbyte/pull/31132) | Fix permission error for `AdAccount` stream | +| 1.1.14 | 2023-09-26 | [30758](https://github.com/airbytehq/airbyte/pull/30758) | Exception should not be raises if a stream is not found | +| 1.1.13 | 2023-09-22 | [30706](https://github.com/airbytehq/airbyte/pull/30706) | Performance testing - include socat binary in docker image | +| 1.1.12 | 2023-09-22 | [30655](https://github.com/airbytehq/airbyte/pull/30655) | Updated doc; improved schema for custom insight streams; updated SAT or custom insight streams; removed obsolete optional max_batch_size option from spec | +| 1.1.11 | 2023-09-21 | [30650](https://github.com/airbytehq/airbyte/pull/30650) | Fix None issue since start_date is optional | +| 1.1.10 | 2023-09-15 | [30485](https://github.com/airbytehq/airbyte/pull/30485) | added 'status' and 'configured_status' fields for campaigns stream schema | +| 1.1.9 | 2023-08-31 | [29994](https://github.com/airbytehq/airbyte/pull/29994) | Removed batch processing, updated description in specs, added user-friendly error message, removed start_date from required attributes | +| 1.1.8 | 2023-09-04 | [29666](https://github.com/airbytehq/airbyte/pull/29666) | Adding custom field `boosted_object_id` to a streams schema in `campaigns` catalog `CustomAudiences` | +| 1.1.7 | 2023-08-21 | [29674](https://github.com/airbytehq/airbyte/pull/29674) | Exclude `rule` from stream `CustomAudiences` | +| 1.1.6 | 2023-08-18 | [29642](https://github.com/airbytehq/airbyte/pull/29642) | Stop batch requests if only 1 left in a batch | +| 1.1.5 | 2023-08-18 | [29610](https://github.com/airbytehq/airbyte/pull/29610) | Automatically reduce batch size | +| 1.1.4 | 2023-08-08 | [29412](https://github.com/airbytehq/airbyte/pull/29412) | Add new custom_audience stream | +| 1.1.3 | 2023-08-08 | [29208](https://github.com/airbytehq/airbyte/pull/29208) | Add account type validation during check | +| 1.1.2 | 2023-08-03 | [29042](https://github.com/airbytehq/airbyte/pull/29042) | Fix broken `advancedAuth` references for `spec` | +| 1.1.1 | 2023-07-26 | [27996](https://github.com/airbytehq/airbyte/pull/27996) | Remove reference to authSpecification | +| 1.1.0 | 2023-07-11 | [26345](https://github.com/airbytehq/airbyte/pull/26345) | Add new `action_report_time` attribute to `AdInsights` class | +| 1.0.1 | 2023-07-07 | [27979](https://github.com/airbytehq/airbyte/pull/27979) | Added the ability to restore the reduced request record limit after the successful retry, and handle the `unknown error` (code 99) with the retry strategy | +| 1.0.0 | 2023-07-05 | [27563](https://github.com/airbytehq/airbyte/pull/27563) | Migrate to FB SDK version 17 | +| 0.5.0 | 2023-06-26 | [27728](https://github.com/airbytehq/airbyte/pull/27728) | License Update: Elv2 | +| 0.4.3 | 2023-05-12 | [27483](https://github.com/airbytehq/airbyte/pull/27483) | Reduce replication start date by one more day | +| 0.4.2 | 2023-06-09 | [27201](https://github.com/airbytehq/airbyte/pull/27201) | Add `complete_oauth_server_output_specification` to spec | +| 0.4.1 | 2023-06-02 | [26941](https://github.com/airbytehq/airbyte/pull/26941) | Remove `authSpecification` from spec.json, use `advanced_auth` instead | +| 0.4.0 | 2023-05-29 | [26720](https://github.com/airbytehq/airbyte/pull/26720) | Add Prebuilt Ad Insights reports | +| 0.3.7 | 2023-05-12 | [26000](https://github.com/airbytehq/airbyte/pull/26000) | Handle config errors | +| 0.3.6 | 2023-04-27 | [22999](https://github.com/airbytehq/airbyte/pull/22999) | Specified date formatting in specification | +| 0.3.5 | 2023-04-26 | [24994](https://github.com/airbytehq/airbyte/pull/24994) | Emit stream status messages | +| 0.3.4 | 2023-04-18 | [22990](https://github.com/airbytehq/airbyte/pull/22990) | Increase pause interval | +| 0.3.3 | 2023-04-14 | [25204](https://github.com/airbytehq/airbyte/pull/25204) | Fix data retention period validation | +| 0.3.2 | 2023-04-08 | [25003](https://github.com/airbytehq/airbyte/pull/25003) | Don't fetch `thumbnail_data_url` if it's None | +| 0.3.1 | 2023-03-27 | [24600](https://github.com/airbytehq/airbyte/pull/24600) | Reduce request record limit when retrying second page or further | +| 0.3.0 | 2023-03-16 | [19141](https://github.com/airbytehq/airbyte/pull/19141) | Added Level parameter to custom Ads Insights | +| 0.2.86 | 2023-03-01 | [23625](https://github.com/airbytehq/airbyte/pull/23625) | Add user friendly fields description in spec and docs. Extend error message for invalid Account ID case. | +| 0.2.85 | 2023-02-14 | [23003](https://github.com/airbytehq/airbyte/pull/23003) | Bump facebook_business to 16.0.0 | +| 0.2.84 | 2023-01-27 | [22003](https://github.com/airbytehq/airbyte/pull/22003) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.2.83 | 2023-01-13 | [21149](https://github.com/airbytehq/airbyte/pull/21149) | Videos stream remove filtering | +| 0.2.82 | 2023-01-09 | [21149](https://github.com/airbytehq/airbyte/pull/21149) | Fix AdAccount schema | +| 0.2.81 | 2023-01-05 | [21057](https://github.com/airbytehq/airbyte/pull/21057) | Remove unsupported fields from request | +| 0.2.80 | 2022-12-21 | [20736](https://github.com/airbytehq/airbyte/pull/20736) | Fix update next cursor | +| 0.2.79 | 2022-12-07 | [20402](https://github.com/airbytehq/airbyte/pull/20402) | Exclude Not supported fields from request | +| 0.2.78 | 2022-12-07 | [20165](https://github.com/airbytehq/airbyte/pull/20165) | Fix fields permission error | +| 0.2.77 | 2022-12-06 | [20131](https://github.com/airbytehq/airbyte/pull/20131) | Update next cursor value at read start | +| 0.2.76 | 2022-12-03 | [20043](https://github.com/airbytehq/airbyte/pull/20043) | Allows `action_breakdowns` to be an empty list - bugfix for #20016 | +| 0.2.75 | 2022-12-03 | [20016](https://github.com/airbytehq/airbyte/pull/20016) | Allows `action_breakdowns` to be an empty list | +| 0.2.74 | 2022-11-25 | [19803](https://github.com/airbytehq/airbyte/pull/19803) | New default for `action_breakdowns`, improve "check" command speed | +| 0.2.73 | 2022-11-21 | [19645](https://github.com/airbytehq/airbyte/pull/19645) | Check "breakdowns" combinations | +| 0.2.72 | 2022-11-04 | [18971](https://github.com/airbytehq/airbyte/pull/18971) | Handle FacebookBadObjectError for empty results on async jobs | +| 0.2.71 | 2022-10-31 | [18734](https://github.com/airbytehq/airbyte/pull/18734) | Reduce request record limit on retry | +| 0.2.70 | 2022-10-26 | [18045](https://github.com/airbytehq/airbyte/pull/18045) | Upgrade FB SDK to v15.0 | +| 0.2.69 | 2022-10-17 | [18045](https://github.com/airbytehq/airbyte/pull/18045) | Remove "pixel" field from the Custom Conversions stream schema | +| 0.2.68 | 2022-10-12 | [17869](https://github.com/airbytehq/airbyte/pull/17869) | Remove "format" from optional datetime `end_date` field | +| 0.2.67 | 2022-10-04 | [17551](https://github.com/airbytehq/airbyte/pull/17551) | Add `cursor_field` for custom_insights stream schema | +| 0.2.65 | 2022-09-29 | [17371](https://github.com/airbytehq/airbyte/pull/17371) | Fix stream CustomConversions `enable_deleted=False` | +| 0.2.64 | 2022-09-22 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | +| 0.2.64 | 2022-09-22 | [17027](https://github.com/airbytehq/airbyte/pull/17027) | Limit time range with 37 months when creating an insight job from lower edge object. Retry bulk request when getting error code `960` | +| 0.2.63 | 2022-09-06 | [15724](https://github.com/airbytehq/airbyte/pull/15724) | Add the Custom Conversion stream | +| 0.2.62 | 2022-09-01 | [16222](https://github.com/airbytehq/airbyte/pull/16222) | Remove `end_date` from config if empty value (re-implement #16096) | +| 0.2.61 | 2022-08-29 | [16096](https://github.com/airbytehq/airbyte/pull/16096) | Remove `end_date` from config if empty value | +| 0.2.60 | 2022-08-19 | [15788](https://github.com/airbytehq/airbyte/pull/15788) | Retry FacebookBadObjectError | +| 0.2.59 | 2022-08-04 | [15327](https://github.com/airbytehq/airbyte/pull/15327) | Shift date validation from config validation to stream method | +| 0.2.58 | 2022-07-25 | [15012](https://github.com/airbytehq/airbyte/pull/15012) | Add `DATA_RETENTION_PERIOD`validation and fix `failed_delivery_checks` field schema type issue | +| 0.2.57 | 2022-07-25 | [14831](https://github.com/airbytehq/airbyte/pull/14831) | Update Facebook SDK to version 14.0.0 | +| 0.2.56 | 2022-07-19 | [14831](https://github.com/airbytehq/airbyte/pull/14831) | Add future `start_date` and `end_date` validation | +| 0.2.55 | 2022-07-18 | [14786](https://github.com/airbytehq/airbyte/pull/14786) | Check if the authorized user has the "MANAGE" task permission when getting the `funding_source_details` field in the ad_account stream | +| 0.2.54 | 2022-06-29 | [14267](https://github.com/airbytehq/airbyte/pull/14267) | Make MAX_BATCH_SIZE available in config | +| 0.2.53 | 2022-06-16 | [13623](https://github.com/airbytehq/airbyte/pull/13623) | Add fields `bid_amount` `bid_strategy` `bid_constraints` to `ads_set` stream | +| 0.2.52 | 2022-06-14 | [13749](https://github.com/airbytehq/airbyte/pull/13749) | Fix the `not syncing any data` issue | +| 0.2.51 | 2022-05-30 | [13317](https://github.com/airbytehq/airbyte/pull/13317) | Change tax_id to string (Canadian has letter in tax_id) | +| 0.2.50 | 2022-04-27 | [12402](https://github.com/airbytehq/airbyte/pull/12402) | Add lookback window to insights streams | +| 0.2.49 | 2022-05-20 | [13047](https://github.com/airbytehq/airbyte/pull/13047) | Fix duplicating records during insights lookback period | +| 0.2.48 | 2022-05-19 | [13008](https://github.com/airbytehq/airbyte/pull/13008) | Update CDK to v0.1.58 avoid crashing on incorrect stream schemas | +| 0.2.47 | 2022-05-06 | [12685](https://github.com/airbytehq/airbyte/pull/12685) | Update CDK to v0.1.56 to emit an `AirbyeTraceMessage` on uncaught exceptions | +| 0.2.46 | 2022-04-22 | [12171](https://github.com/airbytehq/airbyte/pull/12171) | Allow configuration of page_size for requests | +| 0.2.45 | 2022-05-03 | [12390](https://github.com/airbytehq/airbyte/pull/12390) | Better retry logic for split-up async jobs | +| 0.2.44 | 2022-04-14 | [11751](https://github.com/airbytehq/airbyte/pull/11751) | Update API to a directly initialise an AdAccount with the given ID | +| 0.2.43 | 2022-04-13 | [11801](https://github.com/airbytehq/airbyte/pull/11801) | Fix `user_tos_accepted` schema to be an object | +| 0.2.42 | 2022-04-06 | [11761](https://github.com/airbytehq/airbyte/pull/11761) | Upgrade Facebook Python SDK to version 13 | +| 0.2.41 | 2022-03-28 | [11446](https://github.com/airbytehq/airbyte/pull/11446) | Increase number of attempts for individual jobs | +| 0.2.40 | 2022-02-28 | [10698](https://github.com/airbytehq/airbyte/pull/10698) | Improve sleeps time in rate limit handler | +| 0.2.39 | 2022-03-09 | [10917](https://github.com/airbytehq/airbyte/pull/10917) | Retry connections when FB API returns error code 2 (temporary oauth error) | +| 0.2.38 | 2022-03-08 | [10531](https://github.com/airbytehq/airbyte/pull/10531) | Add `time_increment` parameter to custom insights | +| 0.2.37 | 2022-02-28 | [10655](https://github.com/airbytehq/airbyte/pull/10655) | Add Activities stream | +| 0.2.36 | 2022-02-24 | [10588](https://github.com/airbytehq/airbyte/pull/10588) | Fix `execute_in_batch` for large amount of requests | +| 0.2.35 | 2022-02-18 | [10348](https://github.com/airbytehq/airbyte/pull/10348) | Add error code 104 to backoff triggers | +| 0.2.34 | 2022-02-17 | [10180](https://github.com/airbytehq/airbyte/pull/9805) | Performance and reliability fixes | +| 0.2.33 | 2021-12-28 | [10180](https://github.com/airbytehq/airbyte/pull/10180) | Add AdAccount and Images streams | +| 0.2.32 | 2022-01-07 | [10138](https://github.com/airbytehq/airbyte/pull/10138) | Add `primary_key` for all insights streams. | +| 0.2.31 | 2021-12-29 | [9138](https://github.com/airbytehq/airbyte/pull/9138) | Fix videos stream format field incorrect type | +| 0.2.30 | 2021-12-20 | [8962](https://github.com/airbytehq/airbyte/pull/8962) | Add `asset_feed_spec` field to `ad creatives` stream | +| 0.2.29 | 2021-12-17 | [8649](https://github.com/airbytehq/airbyte/pull/8649) | Retrieve ad_creatives image as data encoded | +| 0.2.28 | 2021-12-13 | [8742](https://github.com/airbytehq/airbyte/pull/8742) | Fix for schema generation related to "breakdown" fields | +| 0.2.27 | 2021-11-29 | [8257](https://github.com/airbytehq/airbyte/pull/8257) | Add fields to Campaign stream | +| 0.2.26 | 2021-11-19 | [7855](https://github.com/airbytehq/airbyte/pull/7855) | Add Video stream | +| 0.2.25 | 2021-11-12 | [7904](https://github.com/airbytehq/airbyte/pull/7904) | Implement retry logic for async jobs | +| 0.2.24 | 2021-11-09 | [7744](https://github.com/airbytehq/airbyte/pull/7744) | Fix fail when async job takes too long | +| 0.2.23 | 2021-11-08 | [7734](https://github.com/airbytehq/airbyte/pull/7734) | Resolve $ref field for discover schema | +| 0.2.22 | 2021-11-05 | [7605](https://github.com/airbytehq/airbyte/pull/7605) | Add job retry logics to AdsInsights stream | +| 0.2.21 | 2021-10-05 | [4864](https://github.com/airbytehq/airbyte/pull/4864) | Update insights streams with custom entries for fields, breakdowns and action_breakdowns | +| 0.2.20 | 2021-10-04 | [6719](https://github.com/airbytehq/airbyte/pull/6719) | Update version of facebook_business package to 12.0 | +| 0.2.19 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | +| 0.2.18 | 2021-09-28 | [6499](https://github.com/airbytehq/airbyte/pull/6499) | Fix field values converting fail | +| 0.2.17 | 2021-09-14 | [4978](https://github.com/airbytehq/airbyte/pull/4978) | Convert values' types according to schema types | +| 0.2.16 | 2021-09-14 | [6060](https://github.com/airbytehq/airbyte/pull/6060) | Fix schema for `ads_insights` stream | +| 0.2.15 | 2021-09-14 | [5958](https://github.com/airbytehq/airbyte/pull/5958) | Fix url parsing and add report that exposes conversions | +| 0.2.14 | 2021-07-19 | [4820](https://github.com/airbytehq/airbyte/pull/4820) | Improve the rate limit management | +| 0.2.12 | 2021-06-20 | [3743](https://github.com/airbytehq/airbyte/pull/3743) | Refactor connector to use CDK: - Improve error handling. - Improve async job performance \(insights\). - Add new configuration parameter `insights_days_per_job`. - Rename stream `adsets` to `ad_sets`. - Refactor schema logic for insights, allowing to configure any possible insight stream. | +| 0.2.10 | 2021-06-16 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Update version of facebook_business to 11.0 | +| 0.2.9 | 2021-06-10 | [3996](https://github.com/airbytehq/airbyte/pull/3996) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | +| 0.2.8 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add 80000 as a rate-limiting error code | +| 0.2.7 | 2021-06-03 | [3646](https://github.com/airbytehq/airbyte/pull/3646) | Add missing fields to AdInsights streams | +| 0.2.6 | 2021-05-25 | [3525](https://github.com/airbytehq/airbyte/pull/3525) | Fix handling call rate limit | +| 0.2.5 | 2021-05-20 | [3396](https://github.com/airbytehq/airbyte/pull/3396) | Allow configuring insights lookback window | +| 0.2.4 | 2021-05-13 | [3395](https://github.com/airbytehq/airbyte/pull/3395) | Fix an issue that caused losing Insights data from the past 28 days while incremental sync | +| 0.2.3 | 2021-04-28 | [3116](https://github.com/airbytehq/airbyte/pull/3116) | Wait longer \(5 min\) for async jobs to start | +| 0.2.2 | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726) | Fix base connector versioning | +| 0.2.1 | 2021-03-12 | [2391](https://github.com/airbytehq/airbyte/pull/2391) | Support FB Marketing API v10 | +| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | +| 0.1.4 | 2021-02-24 | [1902](https://github.com/airbytehq/airbyte/pull/1902) | Add `include_deleted` option in params | +| 0.1.3 | 2021-02-15 | [1990](https://github.com/airbytehq/airbyte/pull/1990) | Support Insights stream via async queries | +| 0.1.2 | 2021-01-22 | [1699](https://github.com/airbytehq/airbyte/pull/1699) | Add incremental support | +| 0.1.1 | 2021-01-15 | [1552](https://github.com/airbytehq/airbyte/pull/1552) | Release Native Facebook Marketing Connector |