🎉 Source FB Marketing: add time_increment parameter to custom insig…

…ht streams (#10531) * added time_increment parameter * fix custom insights Co-authored-by: Eugene Kulak <kulak.eugene@gmail.com> Co-authored-by: Sherif A. Nada <snadalive@gmail.com>
airbytehq · Mar 8, 2022 · 2932039 · 2932039
1 parent 63af98e
commit 2932039
Show file tree

Hide file tree

Showing 18 changed files with 440 additions and 208 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -203,7 +203,7 @@
 - name: Facebook Marketing
   sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
   dockerRepository: airbyte/source-facebook-marketing
-  dockerImageTag: 0.2.37
+  dockerImageTag: 0.2.38
   documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing
   icon: facebook.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -1661,7 +1661,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-facebook-marketing:0.2.37"
+- dockerImage: "airbyte/source-facebook-marketing:0.2.38"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"
     changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"
@@ -1737,6 +1737,7 @@
                 description: "The name value of insight"
                 type: "string"
               fields:
+                title: "Fields"
                 description: "A list of chosen fields for fields parameter"
                 default: []
                 type: "array"
@@ -1876,6 +1877,7 @@
                   - "website_purchase_roas"
                   - "wish_bid"
               breakdowns:
+                title: "Breakdowns"
                 description: "A list of chosen breakdowns for breakdowns"
                 default: []
                 type: "array"
@@ -1909,6 +1911,7 @@
                   - "title_asset"
                   - "video_asset"
               action_breakdowns:
+                title: "Action Breakdowns"
                 description: "A list of chosen action_breakdowns for action_breakdowns"
                 default: []
                 type: "array"
@@ -1927,6 +1930,40 @@
                   - "action_type"
                   - "action_video_sound"
                   - "action_video_type"
+              time_increment:
+                title: "Time Increment"
+                description: "Time window in days by which to aggregate statistics.\
+                  \ The sync will be chunked into N day intervals, where N is the\
+                  \ number of days you specified. For example, if you set this value\
+                  \ to 7, then all statistics will be reported as 7-day aggregates\
+                  \ by starting from the start_date. If the start and end dates are\
+                  \ October 1st and October 30th, then the connector will output 5\
+                  \ records: 01 - 06, 07 - 13, 14 - 20, 21 - 27, and 28 - 30 (3 days\
+                  \ only)."
+                default: 1
+                exclusiveMaximum: 90
+                exclusiveMinimum: 0
+                type: "integer"
+              start_date:
+                title: "Start Date"
+                description: "The date from which you'd like to replicate data for\
+                  \ this stream, in the format YYYY-MM-DDT00:00:00Z."
+                pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+                examples:
+                - "2017-01-25T00:00:00Z"
+                type: "string"
+                format: "date-time"
+              end_date:
+                title: "End Date"
+                description: "The date until which you'd like to replicate data for\
+                  \ this stream, in the format YYYY-MM-DDT00:00:00Z. All data generated\
+                  \ between the start date and this date will be replicated. Not setting\
+                  \ this option will result in always syncing the latest data."
+                pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+                examples:
+                - "2017-01-26T00:00:00Z"
+                type: "string"
+                format: "date-time"
             required:
             - "name"
       required:

diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.37
+LABEL io.airbyte.version=0.2.38
 LABEL io.airbyte.name=airbyte/source-facebook-marketing
diff --git a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml
@@ -16,7 +16,6 @@ tests:
       empty_streams: ["videos"]
   incremental:
     - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog_without_insights.json"
       future_state_path: "integration_tests/future_state.json"
   full_refresh:
     - config_path: "secrets/config.json"
diff --git a/...tegrations/connectors/source-facebook-marketing/integration_tests/configured_catalog.json b/...tegrations/connectors/source-facebook-marketing/integration_tests/configured_catalog.json
@@ -2,15 +2,45 @@
   "streams": [
     {
       "stream": {
-        "name": "images",
+        "name": "ad_account",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": null,
+        "default_cursor_field": null,
+        "source_defined_primary_key": [["id"]],
+        "namespace": null
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": null,
+      "destination_sync_mode": "append",
+      "primary_key": null
+    },
+    {
+      "stream": {
+        "name": "ad_sets",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
         "default_cursor_field": ["updated_time"],
         "source_defined_primary_key": [["id"]],
         "namespace": null
       },
-      "sync_mode": "incremental",
+      "sync_mode": "full_refresh",
+      "cursor_field": null,
+      "destination_sync_mode": "append",
+      "primary_key": null
+    },
+    {
+      "stream": {
+        "name": "ads",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["updated_time"],
+        "source_defined_primary_key": [["id"]],
+        "namespace": null
+      },
+      "sync_mode": "full_refresh",
       "cursor_field": null,
       "destination_sync_mode": "append",
       "primary_key": null
@@ -34,29 +64,206 @@
       "stream": {
         "name": "ads_insights",
         "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
         "default_cursor_field": ["date_start"],
+        "source_defined_primary_key": [
+          ["date_start"],
+          ["account_id"],
+          ["ad_id"]
+        ],
+        "namespace": null
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": null,
+      "destination_sync_mode": "append",
+      "primary_key": null
+    },
+    {
+      "stream": {
+        "name": "ads_insights_age_and_gender",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["date_start"],
+        "source_defined_primary_key": [
+          ["date_start"],
+          ["account_id"],
+          ["ad_id"],
+          ["age"],
+          ["gender"]
+        ],
+        "namespace": null
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": null,
+      "destination_sync_mode": "append",
+      "primary_key": null
+    },
+    {
+      "stream": {
+        "name": "ads_insights_country",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["date_start"],
+        "source_defined_primary_key": [
+          ["date_start"],
+          ["account_id"],
+          ["ad_id"],
+          ["country"]
+        ],
+        "namespace": null
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": null,
+      "destination_sync_mode": "append",
+      "primary_key": null
+    },
+    {
+      "stream": {
+        "name": "ads_insights_region",
+        "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "source_defined_primary_key": []
+        "default_cursor_field": ["date_start"],
+        "source_defined_primary_key": [
+          ["date_start"],
+          ["account_id"],
+          ["ad_id"],
+          ["region"]
+        ],
+        "namespace": null
       },
-      "sync_mode": "incremental",
-      "primary_key": [],
-      "cursor_field": ["date_start"],
-      "destination_sync_mode": "append"
+      "sync_mode": "full_refresh",
+      "cursor_field": null,
+      "destination_sync_mode": "append",
+      "primary_key": null
+    },
+    {
+      "stream": {
+        "name": "ads_insights_dma",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["date_start"],
+        "source_defined_primary_key": [
+          ["date_start"],
+          ["account_id"],
+          ["ad_id"],
+          ["dma"]
+        ],
+        "namespace": null
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": null,
+      "destination_sync_mode": "append",
+      "primary_key": null
     },
     {
       "stream": {
         "name": "ads_insights_platform_and_device",
         "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["date_start"],
+        "source_defined_primary_key": [
+          ["date_start"],
+          ["account_id"],
+          ["ad_id"],
+          ["publisher_platform"],
+          ["platform_position"],
+          ["impression_device"]
+        ],
+        "namespace": null
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": null,
+      "destination_sync_mode": "append",
+      "primary_key": null
+    },
+    {
+      "stream": {
+        "name": "ads_insights_action_type",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
         "default_cursor_field": ["date_start"],
+        "source_defined_primary_key": [
+          ["date_start"],
+          ["account_id"],
+          ["ad_id"]
+        ],
+        "namespace": null
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": null,
+      "destination_sync_mode": "append",
+      "primary_key": null
+    },
+    {
+      "stream": {
+        "name": "campaigns",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["updated_time"],
+        "source_defined_primary_key": [["id"]],
+        "namespace": null
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": null,
+      "destination_sync_mode": "append",
+      "primary_key": null
+    },
+    {
+      "stream": {
+        "name": "images",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["updated_time"],
+        "source_defined_primary_key": [["id"]],
+        "namespace": null
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": null,
+      "destination_sync_mode": "append",
+      "primary_key": null
+    },
+    {
+      "stream": {
+        "name": "videos",
+        "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "source_defined_primary_key": []
+        "default_cursor_field": ["updated_time"],
+        "source_defined_primary_key": [["id"]],
+        "namespace": null
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": null,
+      "destination_sync_mode": "append",
+      "primary_key": null
+    },
+    {
+      "stream": {
+        "name": "custommy_custom_insights",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["date_start"],
+        "source_defined_primary_key": [
+          ["date_start"],
+          ["account_id"],
+          ["ad_id"]
+        ],
+        "namespace": null
       },
-      "sync_mode": "incremental",
-      "primary_key": [],
-      "cursor_field": ["date_start"],
-      "destination_sync_mode": "append"
+      "sync_mode": "full_refresh",
+      "cursor_field": null,
+      "destination_sync_mode": "append",
+      "primary_key": null
     }
   ]
 }