🐛 Source Facebook Marketing: remove "end_date" from config if empty v…

…alue (re-implement #16096) (#16222)

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -256,7 +256,7 @@
 - name: Facebook Marketing
   sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
   dockerRepository: airbyte/source-facebook-marketing
-  dockerImageTag: 0.2.61
+  dockerImageTag: 0.2.62
   documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing
   icon: facebook.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -1857,7 +1857,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-facebook-marketing:0.2.61"
+- dockerImage: "airbyte/source-facebook-marketing:0.2.62"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"
     changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"
@@ -1891,7 +1891,7 @@
             \ between start_date and this date will be replicated. Not setting this\
             \ option will result in always syncing the latest data."
           order: 2
-          pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+          pattern: "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
           examples:
           - "2017-01-26T00:00:00Z"
           type: "string"
@@ -1939,7 +1939,8 @@
                 type: "array"
                 items:
                   title: "ValidEnums"
-                  description: "An enumeration."
+                  description: "Generic enumeration.\n\nDerive from this class to\
+                    \ define new enumerations."
                   enum:
                   - "account_currency"
                   - "account_id"
@@ -2078,7 +2079,8 @@
                 type: "array"
                 items:
                   title: "ValidBreakdowns"
-                  description: "An enumeration."
+                  description: "Generic enumeration.\n\nDerive from this class to\
+                    \ define new enumerations."
                   enum:
                   - "ad_format_asset"
                   - "age"
@@ -2111,7 +2113,8 @@
                 type: "array"
                 items:
                   title: "ValidActionBreakdowns"
-                  description: "An enumeration."
+                  description: "Generic enumeration.\n\nDerive from this class to\
+                    \ define new enumerations."
                   enum:
                   - "action_canvas_component_name"
                   - "action_carousel_card_id"

airbyte-integrations/connectors/source-facebook-marketing/Dockerfile

@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=0.2.61
+LABEL io.airbyte.version=0.2.62
 LABEL io.airbyte.name=airbyte/source-facebook-marketing

airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json

@@ -25,7 +25,7 @@
         "title": "End Date",
         "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the latest data.",
         "order": 2,
-        "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
+        "pattern": "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
         "examples": ["2017-01-26T00:00:00Z"],
         "type": "string",
         "format": "date-time"
@@ -73,7 +73,7 @@
               "type": "array",
               "items": {
                 "title": "ValidEnums",
-                "description": "An enumeration.",
+                "description": "Generic enumeration.\n\nDerive from this class to define new enumerations.",
                 "enum": [
                   "account_currency",
                   "account_id",
@@ -215,7 +215,7 @@
               "type": "array",
               "items": {
                 "title": "ValidBreakdowns",
-                "description": "An enumeration.",
+                "description": "Generic enumeration.\n\nDerive from this class to define new enumerations.",
                 "enum": [
                   "ad_format_asset",
                   "age",
@@ -251,7 +251,7 @@
               "type": "array",
               "items": {
                 "title": "ValidActionBreakdowns",
-                "description": "An enumeration.",
+                "description": "Generic enumeration.\n\nDerive from this class to define new enumerations.",
                 "enum": [
                   "action_canvas_component_name",
                   "action_carousel_card_id",

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py

@@ -3,12 +3,10 @@
 #
 
 import logging
-import os
 from typing import Any, List, Mapping, Optional, Tuple, Type
 
 import pendulum
 import requests
-from airbyte_cdk.connector import _WriteConfigProtocol
 from airbyte_cdk.models import AuthSpecification, ConnectorSpecification, DestinationSyncMode, OAuth2Specification
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
@@ -38,16 +36,12 @@
 
 
 class SourceFacebookMarketing(AbstractSource):
-    def configure(self: _WriteConfigProtocol, config: Mapping[str, Any], temp_dir: str) -> Mapping[str, Any]:
-        source_spec = self.spec(logging.getLogger("airbyte"))
-        end_date = source_spec.connectionSpecification["properties"]["end_date"]
-        # We highlight here that "end_date" is not a simple "string" field it's an extended type with "format" modifier.
-        # If "end_date" is provided as an empty string we can treat this case as missed value.
-        if end_date["type"] == "string" and "format" in end_date:
-            if config.get("end_date") == "":
-                config.pop("end_date")
-        config_path = os.path.join(temp_dir, "config.json")
-        self.write_config(config, config_path)
+    def _validate_and_transform(self, config: Mapping[str, Any]):
+        if config.get("end_date") == "":
+            config.pop("end_date")
+        config = ConnectorConfig.parse_obj(config)
+        config.start_date = pendulum.instance(config.start_date)
+        config.end_date = pendulum.instance(config.end_date)
         return config
 
     def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
@@ -57,9 +51,10 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) ->
         :param config:  the user-input config object conforming to the connector's spec.json
         :return Tuple[bool, Any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise.
         """
-        config = ConnectorConfig.parse_obj(config)
-        if pendulum.instance(config.end_date) < pendulum.instance(config.start_date):
-            raise ValueError("end_date must be equal or after start_date.")
+        config = self._validate_and_transform(config)
+        if config.end_date < config.start_date:
+            return False, "end_date must be equal or after start_date."
+
         try:
             api = API(account_id=config.account_id, access_token=config.access_token)
             logger.info(f"Select account {api.account}")
@@ -73,8 +68,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]:
         :param config: A Mapping of the user input configuration as defined in the connector spec.
         :return: list of the stream instances
         """
-        config: ConnectorConfig = ConnectorConfig.parse_obj(config)
-
+        config = self._validate_and_transform(config)
         config.start_date = validate_start_date(config.start_date)
         config.end_date = validate_end_date(config.start_date, config.end_date)
 

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py

@@ -3,11 +3,10 @@
 #
 
 import logging
-from datetime import datetime
+from datetime import datetime, timezone
 from enum import Enum
 from typing import List, Optional
 
-import pendulum
 from airbyte_cdk.sources.config import BaseConfig
 from facebook_business.adobjects.adsinsights import AdsInsights
 from pydantic import BaseModel, Field, PositiveInt
@@ -19,6 +18,7 @@
 ValidBreakdowns = Enum("ValidBreakdowns", AdsInsights.Breakdowns.__dict__)
 ValidActionBreakdowns = Enum("ValidActionBreakdowns", AdsInsights.ActionBreakdowns.__dict__)
 DATE_TIME_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+EMPTY_PATTERN = "^$"
 
 
 class InsightConfig(BaseModel):
@@ -118,9 +118,9 @@ class Config:
             "All data generated between start_date and this date will be replicated. "
             "Not setting this option will result in always syncing the latest data."
         ),
-        pattern=DATE_TIME_PATTERN,
+        pattern=EMPTY_PATTERN + "|" + DATE_TIME_PATTERN,
         examples=["2017-01-26T00:00:00Z"],
-        default_factory=pendulum.now,
+        default_factory=lambda: datetime.now(tz=timezone.utc),
     )
 
     access_token: str = Field(

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py

@@ -3,9 +3,9 @@
 #
 
 import logging
-from datetime import datetime
 
 import pendulum
+from pendulum import DateTime
 
 logger = logging.getLogger("airbyte")
 
@@ -16,26 +16,25 @@
 DATA_RETENTION_PERIOD = 37
 
 
-def validate_start_date(start_date: datetime) -> datetime:
-    pendulum_date = pendulum.instance(start_date)
-    time_zone = start_date.tzinfo
-    current_date = pendulum.today(time_zone)
-    if pendulum_date.timestamp() > pendulum.now().timestamp():
-        message = f"The start date cannot be in the future. Set start date to today's date - {current_date}."
+def validate_start_date(start_date: DateTime) -> DateTime:
+    now = pendulum.now(tz=start_date.tzinfo)
+    today = now.replace(microsecond=0, second=0, minute=0, hour=0)
+    retention_date = today.subtract(months=DATA_RETENTION_PERIOD)
+
+    if start_date > now:
+        message = f"The start date cannot be in the future. Set start date to today's date - {today}."
         logger.warning(message)
-        return current_date
-    elif pendulum_date.timestamp() < current_date.subtract(months=DATA_RETENTION_PERIOD).timestamp():
-        current_date = pendulum.today(time_zone)
+        return today
+    elif start_date < retention_date:
         message = (
-            f"The start date cannot be beyond {DATA_RETENTION_PERIOD} months from the current date. "
-            f"Set start date to {current_date.subtract(months=DATA_RETENTION_PERIOD)}."
+            f"The start date cannot be beyond {DATA_RETENTION_PERIOD} months from the current date. Set start date to {retention_date}."
         )
         logger.warning(message)
-        return current_date.subtract(months=DATA_RETENTION_PERIOD)
+        return retention_date
     return start_date
 
 
-def validate_end_date(start_date: datetime, end_date: datetime) -> datetime:
+def validate_end_date(start_date: DateTime, end_date: DateTime) -> DateTime:
     if start_date > end_date:
         message = f"The end date must be after start date. Set end date to {start_date}."
         logger.warning(message)

airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py

@@ -2,25 +2,42 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
+
+from copy import deepcopy
+
 import pydantic
 import pytest
-from airbyte_cdk.models import ConnectorSpecification
+from airbyte_cdk.models import AirbyteConnectionStatus, ConnectorSpecification, Status
+from facebook_business import FacebookAdsApi, FacebookSession
 from source_facebook_marketing import SourceFacebookMarketing
 from source_facebook_marketing.spec import ConnectorConfig
 
+from .utils import command_check
+
 
 @pytest.fixture(name="config")
 def config_fixture():
     config = {
-        "account_id": 123,
+        "account_id": "123",
         "access_token": "TOKEN",
-        "start_date": "2019-10-10T00:00:00",
-        "end_date": "2020-10-10T00:00:00",
+        "start_date": "2019-10-10T00:00:00Z",
+        "end_date": "2020-10-10T00:00:00Z",
     }
 
     return config
 
 
+@pytest.fixture
+def config_gen(config):
+    def inner(**kwargs):
+        new_config = deepcopy(config)
+        # WARNING, no support deep dictionaries
+        new_config.update(kwargs)
+        return {k: v for k, v in new_config.items() if v is not ...}
+
+    return inner
+
+
 @pytest.fixture(name="api")
 def api_fixture(mocker):
     api_mock = mocker.patch("source_facebook_marketing.source.API")
@@ -45,9 +62,10 @@ def test_check_connection_ok(self, api, config, logger_mock):
     def test_check_connection_end_date_before_start_date(self, api, config, logger_mock):
         config["start_date"] = "2019-10-10T00:00:00"
         config["end_date"] = "2019-10-09T00:00:00"
-
-        with pytest.raises(ValueError, match="end_date must be equal or after start_date."):
-            SourceFacebookMarketing().check_connection(logger_mock, config=config)
+        assert SourceFacebookMarketing().check_connection(logger_mock, config=config) == (
+            False,
+            "end_date must be equal or after start_date.",
+        )
 
     def test_check_connection_empty_config(self, api, logger_mock):
         config = {}
@@ -95,3 +113,22 @@ def test_update_insights_streams(self, api, config):
         assert SourceFacebookMarketing()._update_insights_streams(
             insights=config.custom_insights, default_args=insights_args, streams=streams
         )
+
+
+def test_check_config(config_gen, requests_mock):
+    requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FacebookAdsApi.API_VERSION}/act_123/", {})
+
+    source = SourceFacebookMarketing()
+    assert command_check(source, config_gen()) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None)
+
+    status = command_check(source, config_gen(start_date="2019-99-10T00:00:00Z"))
+    assert status.status == Status.FAILED
+
+    status = command_check(source, config_gen(end_date="2019-99-10T00:00:00Z"))
+    assert status.status == Status.FAILED
+
+    with pytest.raises(Exception):
+        assert command_check(source, config_gen(start_date=...))
+
+    assert command_check(source, config_gen(end_date=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None)
+    assert command_check(source, config_gen(end_date="")) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None)

airbyte-integrations/connectors/source-facebook-marketing/unit_tests/utils.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+
+from unittest import mock
+
+from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification
+from airbyte_cdk.sources import Source
+from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config
+
+
+def command_check(source: Source, config):
+    logger = mock.MagicMock()
+    connector_config, _ = split_config(config)
+    if source.check_config_against_spec:
+        source_spec: ConnectorSpecification = source.spec(logger)
+        check_config_against_spec_or_exit(connector_config, source_spec)
+    return source.check(logger, config)

docs/integrations/sources/facebook-marketing.md

@@ -120,6 +120,7 @@ Please be informed that the connector uses the `lookback_window` parameter to pe
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                                                                                                                                                                           |
 |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.2.62  | 2022-09-01 | [16222](https://github.com/airbytehq/airbyte/pull/16222) | Remove `end_date` from config if empty value (re-implement #16096)                                                                                                                                                                                                                                |
 | 0.2.61  | 2022-08-29 | [16096](https://github.com/airbytehq/airbyte/pull/16096) | Remove `end_date` from config if empty value                                                                                                                                                                                                                                                      |
 | 0.2.60  | 2022-08-19 | [15788](https://github.com/airbytehq/airbyte/pull/15788) | Retry FacebookBadObjectError                                                                                                                                                                                                                                                                      |
 | 0.2.59  | 2022-08-04 | [15327](https://github.com/airbytehq/airbyte/pull/15327) | Shift date validation from config validation to stream method                                                                                                                                                                                                                                     |