🎉 Source Facebook Marketing - remove batch processing, add user frien…

…dly config errors (#29994)

Co-authored-by: midavadim <midavadim@users.noreply.github.com>

airbyte-integrations/connectors/source-facebook-marketing/Dockerfile

@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=1.1.8
+LABEL io.airbyte.version=1.1.9
 LABEL io.airbyte.name=airbyte/source-facebook-marketing

airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml

@@ -7,7 +7,8 @@ acceptance_tests:
     tests:
       - spec_path: "integration_tests/spec.json"
         backward_compatibility_tests_config:
-          disable_for_version: "0.5.0"
+          disable_for_version: "1.1.9"
+          previous_connector_version: "1.1.8"
   connection:
     tests:
       - config_path: "secrets/config.json"
@@ -46,6 +47,35 @@ acceptance_tests:
           ads_insights_dma:
             - name: cost_per_estimated_ad_recallers
               bypass_reason: can be missing
+          ads_insights_age_and_gender:
+            - name: cost_per_estimated_ad_recallers
+              bypass_reason: can be missing
+          ads_insights_delivery_device:
+            - name: cost_per_estimated_ad_recallers
+              bypass_reason: can be missing
+          ads_insights_delivery_platform_and_device_platform:
+            - name: cost_per_estimated_ad_recallers
+              bypass_reason: can be missing
+          ads_insights_demographics_age:
+            - name: cost_per_estimated_ad_recallers
+              bypass_reason: can be missing
+          ads_insights_demographics_country:
+            - name: cost_per_estimated_ad_recallers
+              bypass_reason: can be missing
+          ads_insights_demographics_gender:
+            - name: cost_per_estimated_ad_recallers
+              bypass_reason: can be missing
+          ads_insights_platform_and_device:
+            - name: cost_per_estimated_ad_recallers
+              bypass_reason: can be missing
+          ads_insights_region:
+            - name: cost_per_estimated_ad_recallers
+              bypass_reason: can be missing
+          custom_audiences:
+            - name: approximate_count_lower_bound
+              bypass_reason: is changeable
+            - name: approximate_count_upper_bound
+              bypass_reason: is changeable
         empty_streams:
           - name: "ads_insights_action_product_id"
             bypass_reason: "Data not permanent"

airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json

@@ -6,18 +6,25 @@
     "type": "object",
     "properties": {
       "account_id": {
-        "title": "Account ID",
-        "description": "The Facebook Ad account ID to use when pulling data from the Facebook Marketing API. Open your Meta Ads Manager. The Ad account ID number is in the account dropdown menu or in your browser's address bar. See the <a href=\"https://www.facebook.com/business/help/1492627900875762\">docs</a> for more information.",
+        "title": "Ad Account ID",
+        "description": "The Facebook Ad account ID to use when pulling data from the Facebook Marketing API. The Ad account ID number is in the account dropdown menu or in your browser's address bar of your <a href=\"https://adsmanager.facebook.com/adsmanager/\">Meta Ads Manager</a>. See the <a href=\"https://www.facebook.com/business/help/1492627900875762\">docs</a> for more information.",
         "order": 0,
         "pattern": "^[0-9]+$",
         "pattern_descriptor": "1234567890",
         "examples": ["111111111111111"],
         "type": "string"
       },
+      "access_token": {
+        "title": "Access Token",
+        "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions <b>ads_management, ads_read, read_insights, business_management</b>. Then click on \"Get token\". See the <a href=\"https://docs.airbyte.com/integrations/sources/facebook-marketing\">docs</a> for more information.",
+        "order": 1,
+        "airbyte_secret": true,
+        "type": "string"
+      },
       "start_date": {
         "title": "Start Date",
-        "description": "The date from which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.",
-        "order": 1,
+        "description": "The date from which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z. If not set then all data will be replicated for usual streams and only last 2 years for insight streams.",
+        "order": 2,
         "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
         "examples": ["2017-01-25T00:00:00Z"],
         "type": "string",
@@ -26,19 +33,12 @@
       "end_date": {
         "title": "End Date",
         "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z. All data generated between the start date and this end date will be replicated. Not setting this option will result in always syncing the latest data.",
-        "order": 2,
+        "order": 3,
         "pattern": "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
         "examples": ["2017-01-26T00:00:00Z"],
         "type": "string",
         "format": "date-time"
       },
-      "access_token": {
-        "title": "Access Token",
-        "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions <b>ads_management, ads_read, read_insights, business_management</b>. Then click on \"Get token\". See the <a href=\"https://docs.airbyte.com/integrations/sources/facebook-marketing\">docs</a> for more information.",
-        "order": 3,
-        "airbyte_secret": true,
-        "type": "string"
-      },
       "include_deleted": {
         "title": "Include Deleted Campaigns, Ads, and AdSets",
         "description": "Set to active if you want to include data from deleted Campaigns, Ads, and AdSets.",
@@ -378,7 +378,7 @@
         "type": "string"
       }
     },
-    "required": ["account_id", "start_date", "access_token"]
+    "required": ["account_id", "access_token"]
   },
   "supportsIncremental": true,
   "supported_destination_sync_modes": ["append"],

airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml

@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
-  dockerImageTag: 1.1.8
+  dockerImageTag: 1.1.9
   dockerRepository: airbyte/source-facebook-marketing
   githubIssueLabel: source-facebook-marketing
   icon: facebook.svg
@@ -17,6 +17,22 @@ data:
     oss:
       enabled: true
   releaseStage: generally_available
+  suggestedStreams:
+    streams:
+      - ads_insights
+      - campaigns
+      - ads
+      - ad_sets
+      - ad_creatives
+      - ads_insights_age_and_gender
+      - ads_insights_action_type
+      - custom_conversions
+      - images
+      - ads_insights_country
+      - ads_insights_platform_and_device
+      - ads_insights_region
+      - ads_insights_dma
+      - activities
   documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing
   tags:
     - language:python

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py

@@ -9,8 +9,6 @@
 
 import backoff
 import pendulum
-from airbyte_cdk.models import FailureType
-from airbyte_cdk.utils import AirbyteTracedException
 from cached_property import cached_property
 from facebook_business import FacebookAdsApi
 from facebook_business.adobjects.adaccount import AdAccount
@@ -77,7 +75,6 @@ def _parse_call_rate_header(headers):
             )
 
         if usage_header_business:
-
             usage_header_business_loaded = json.loads(usage_header_business)
             for business_object_id in usage_header_business_loaded:
                 usage_limits = usage_header_business_loaded.get(business_object_id)[0]
@@ -194,15 +191,4 @@ def account(self) -> AdAccount:
     @staticmethod
     def _find_account(account_id: str) -> AdAccount:
         """Actual implementation of find account"""
-        try:
-            return AdAccount(f"act_{account_id}").api_get()
-        except FacebookRequestError as exc:
-            message = (
-                f"Error: {exc.api_error_code()}, {exc.api_error_message()}. "
-                f"Please also verify your Account ID: "
-                f"See the https://www.facebook.com/business/help/1492627900875762 for more information."
-            )
-            raise AirbyteTracedException(
-                message=message,
-                failure_type=FailureType.config_error,
-            ) from exc
+        return AdAccount(f"act_{account_id}").api_get()

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py

@@ -7,20 +7,19 @@
 
 import facebook_business
 import pendulum
-import requests
 from airbyte_cdk.models import (
     AdvancedAuth,
     AuthFlowType,
     ConnectorSpecification,
     DestinationSyncMode,
     FailureType,
     OAuthConfigSpecification,
+    SyncMode,
 )
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.utils import AirbyteTracedException
-from pydantic.error_wrappers import ValidationError
-from source_facebook_marketing.api import API, FacebookAPIException
+from source_facebook_marketing.api import API
 from source_facebook_marketing.spec import ConnectorConfig
 from source_facebook_marketing.streams import (
     Activities,
@@ -54,7 +53,6 @@
     Images,
     Videos,
 )
-from source_facebook_marketing.streams.common import AccountTypeException
 
 from .utils import validate_end_date, validate_start_date
 
@@ -67,9 +65,14 @@ def _validate_and_transform(self, config: Mapping[str, Any]):
         config.setdefault("action_breakdowns_allow_empty", False)
         if config.get("end_date") == "":
             config.pop("end_date")
+
         config = ConnectorConfig.parse_obj(config)
-        config.start_date = pendulum.instance(config.start_date)
-        config.end_date = pendulum.instance(config.end_date)
+
+        if config.start_date:
+            config.start_date = pendulum.instance(config.start_date)
+
+        if config.end_date:
+            config.end_date = pendulum.instance(config.end_date)
         return config
 
     def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
@@ -84,23 +87,26 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) ->
 
             if config.end_date > pendulum.now():
                 return False, "Date range can not be in the future."
-            if config.end_date < config.start_date:
-                return False, "end_date must be equal or after start_date."
+            if config.start_date and config.end_date < config.start_date:
+                return False, "End date must be equal or after start date."
 
             api = API(account_id=config.account_id, access_token=config.access_token, page_size=config.page_size)
-            logger.info(f"Select account {api.account}")
 
-            account_info = api.account.api_get(fields=["is_personal"])
+            record_iterator = AdAccount(api=api).read_records(sync_mode=SyncMode.full_refresh, stream_state={})
+            account_info = list(record_iterator)[0]
 
             if account_info.get("is_personal"):
                 message = (
                     "The personal ad account you're currently using is not eligible "
                     "for this operation. Please switch to a business ad account."
                 )
-                raise AccountTypeException(message)
+                return False, message
 
-        except (requests.exceptions.RequestException, ValidationError, FacebookAPIException, AccountTypeException) as e:
-            return False, e
+        except AirbyteTracedException as e:
+            return False, f"{e.message}. Full error: {e.internal_message}"
+
+        except Exception as e:
+            return False, f"Unexpected error: {repr(e)}"
 
         # make sure that we have valid combination of "action_breakdowns" and "breakdowns" parameters
         for stream in self.get_custom_insights_streams(api, config):
@@ -117,13 +123,17 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]:
         :return: list of the stream instances
         """
         config = self._validate_and_transform(config)
-        config.start_date = validate_start_date(config.start_date)
-        config.end_date = validate_end_date(config.start_date, config.end_date)
+        if config.start_date:
+            config.start_date = validate_start_date(config.start_date)
+            config.end_date = validate_end_date(config.start_date, config.end_date)
 
         api = API(account_id=config.account_id, access_token=config.access_token, page_size=config.page_size)
 
+        # if start_date not specified then set default start_date for report streams to 2 years ago
+        report_start_date = config.start_date or pendulum.now().add(years=-2)
+
         insights_args = dict(
-            api=api, start_date=config.start_date, end_date=config.end_date, insights_lookback_window=config.insights_lookback_window
+            api=api, start_date=report_start_date, end_date=config.end_date, insights_lookback_window=config.insights_lookback_window
         )
         streams = [
             AdAccount(api=api),

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py

@@ -106,32 +106,45 @@ class Config:
         title = "Source Facebook Marketing"
 
     account_id: str = Field(
-        title="Account ID",
+        title="Ad Account ID",
         order=0,
         description=(
-            "The Facebook Ad account ID to use when pulling data from the Facebook Marketing API."
-            " Open your Meta Ads Manager. The Ad account ID number is in the account dropdown menu or in your browser's address bar. "
+            "The Facebook Ad account ID to use when pulling data from the Facebook Marketing API. "
+            "The Ad account ID number is in the account dropdown menu or in your browser's address "
+            'bar of your <a href="https://adsmanager.facebook.com/adsmanager/">Meta Ads Manager</a>. '
             'See the <a href="https://www.facebook.com/business/help/1492627900875762">docs</a> for more information.'
         ),
         pattern="^[0-9]+$",
         pattern_descriptor="1234567890",
         examples=["111111111111111"],
     )
 
-    start_date: datetime = Field(
-        title="Start Date",
+    access_token: str = Field(
+        title="Access Token",
         order=1,
+        description=(
+            "The value of the generated access token. "
+            'From your App’s Dashboard, click on "Marketing API" then "Tools". '
+            'Select permissions <b>ads_management, ads_read, read_insights, business_management</b>. Then click on "Get token". '
+            'See the <a href="https://docs.airbyte.com/integrations/sources/facebook-marketing">docs</a> for more information.'
+        ),
+        airbyte_secret=True,
+    )
+
+    start_date: Optional[datetime] = Field(
+        title="Start Date",
+        order=2,
         description=(
             "The date from which you'd like to replicate data for all incremental streams, "
-            "in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated."
+            "in the format YYYY-MM-DDT00:00:00Z. If not set then all data will be replicated for usual streams and only last 2 years for insight streams."
         ),
         pattern=DATE_TIME_PATTERN,
         examples=["2017-01-25T00:00:00Z"],
     )
 
     end_date: Optional[datetime] = Field(
         title="End Date",
-        order=2,
+        order=3,
         description=(
             "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z."
             " All data generated between the start date and this end date will be replicated. "
@@ -142,18 +155,6 @@ class Config:
         default_factory=lambda: datetime.now(tz=timezone.utc),
     )
 
-    access_token: str = Field(
-        title="Access Token",
-        order=3,
-        description=(
-            "The value of the generated access token. "
-            'From your App’s Dashboard, click on "Marketing API" then "Tools". '
-            'Select permissions <b>ads_management, ads_read, read_insights, business_management</b>. Then click on "Get token". '
-            'See the <a href="https://docs.airbyte.com/integrations/sources/facebook-marketing">docs</a> for more information.'
-        ),
-        airbyte_secret=True,
-    )
-
     include_deleted: bool = Field(
         title="Include Deleted Campaigns, Ads, and AdSets",
         order=4,