Source Amazon Ads: get rid of fail_on_extra_columns: false in SAT (a…

…irbytehq#25913) * Source Amazon Ads: small schema fixes * Source Amazon Ads: update changelog * Source Amazon Ads: update unittest * Source Amazon Ads: unittest additional property is boolean * Source Amazon Ads: bump version * auto-bump connector version --------- Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
natalia-miinto · Jun 8, 2023 · 057916d · 057916d
1 parent 2a9593a
commit 057916d
Show file tree

Hide file tree

Showing 12 changed files with 31 additions and 8 deletions.
diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json b/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json
@@ -8688,7 +8688,7 @@
     "sourceDefinitionId": "c6b0a29e-1da9-4512-9002-7bfd0cba2246",
     "name": "Amazon Ads",
     "dockerRepository": "airbyte/source-amazon-ads",
-    "dockerImageTag": "1.0.5",
+    "dockerImageTag": "1.0.6",
     "documentationUrl": "https://docs.airbyte.com/integrations/sources/amazon-ads",
     "icon": "amazonads.svg",
     "sourceType": "api",

diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml b/airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml
@@ -73,7 +73,7 @@
 - name: Amazon Ads
   sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
   dockerRepository: airbyte/source-amazon-ads
-  dockerImageTag: 1.0.5
+  dockerImageTag: 1.0.6
   documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads
   icon: amazonads.svg
   sourceType: api

diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml b/airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml
@@ -852,7 +852,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-amazon-ads:1.0.5"
+- dockerImage: "airbyte/source-amazon-ads:1.0.6"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile
@@ -14,5 +14,5 @@ ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
 
-LABEL io.airbyte.version=1.0.5
+LABEL io.airbyte.version=1.0.6
 LABEL io.airbyte.name=airbyte/source-amazon-ads
diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml
@@ -28,7 +28,6 @@ acceptance_tests:
         timeout_seconds: 2400
         expect_records:
           path: integration_tests/expected_records.jsonl
-        fail_on_extra_columns: false
   connection:
     tests:
       - config_path: secrets/config.json

diff --git a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml
@@ -8,7 +8,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
-  dockerImageTag: 1.0.5
+  dockerImageTag: 1.0.6
   dockerRepository: airbyte/source-amazon-ads
   githubIssueLabel: source-amazon-ads
   icon: amazonads.svg

diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/common.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/common.py
@@ -36,6 +36,9 @@ def schema_extra(cls, schema: Dict[str, Any], model: Type["BaseModel"]) -> None:
                 if "type" in prop:
                     if allow_none:
                         prop["type"] = ["null", prop["type"]]
+                    if prop["type"] == "array" and prop["items"]:
+                        if prop["items"].pop("additionalProperties", None):
+                            prop["items"]["additionalProperties"] = True
 
 
 class MetricsReport(CatalogModel):

diff --git a/...-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_display.py b/...-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_display.py
@@ -3,6 +3,7 @@
 #
 
 from decimal import Decimal
+from typing import Dict, List
 
 from .common import CatalogModel, Targeting
 
@@ -29,6 +30,7 @@ class DisplayAdGroup(CatalogModel):
     bidOptimization: str
     state: str
     tactic: str
+    creativeType: str
 
 
 class DisplayProductAds(CatalogModel):
@@ -41,4 +43,5 @@ class DisplayProductAds(CatalogModel):
 
 
 class DisplayTargeting(Targeting):
-    pass
+    expression: List[Dict[str, str]]
+    resolvedExpression: List[Dict[str, str]]
diff --git a/...integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py b/...integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py
@@ -27,6 +27,7 @@ class ProductCampaign(CatalogModel):
     targetingType: str
     state: str
     dailyBudget: Decimal
+    ruleBasedBudget: Dict[str, str]
     startDate: str
     endDate: str = None
     premiumBidAdjustment: bool
@@ -52,3 +53,5 @@ class ProductAd(CatalogModel):
 
 class ProductTargeting(Targeting):
     campaignId: Decimal
+    expression: List[Dict[str, str]]
+    resolvedExpression: List[Dict[str, str]]
diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py
@@ -62,7 +62,7 @@ def product_ads_response():
 @fixture
 def targeting_response():
     return """
-[{"targetId":123,"adGroupId":321,"state":"enabled","expressionType":"manual","bid":1.5,"expression":{"type":"asinSameAs","value":"B0123456789"},"resolvedExpression":{"type":"views","values":{"type":"asinCategorySameAs","value":"B0123456789"}}}]
+[{"targetId":123,"adGroupId":321,"state":"enabled","expressionType":"manual","bid":1.5,"expression":[{"type":"asinSameAs","value":"B0123456789"}],"resolvedExpression":[{"type":"views","values":{"type":"asinCategorySameAs","value":"B0123456789"}}]}]
 """
 
 

diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py
@@ -23,6 +23,19 @@ def setup_responses():
     )
 
 
+def ensure_additional_property_is_boolean(root):
+    for name, prop in root.get("properties", {}).items():
+        if prop["type"] == "array" and "items" in prop:
+            ensure_additional_property_is_boolean(prop["items"])
+        if prop["type"] == "object" and "properties" in prop:
+            ensure_additional_property_is_boolean(prop)
+    if "additionalProperties" in root:
+        assert type(root["additionalProperties"]) == bool, (
+            f"`additionalProperties` expected to be of 'bool' type. "
+            f"Got: {type(root['additionalProperties']).__name__}"
+        )
+
+
 @responses.activate
 def test_discover(config):
     setup_responses()
@@ -32,6 +45,7 @@ def test_discover(config):
     schemas = [stream["json_schema"] for stream in catalog["catalog"]["streams"]]
     for schema in schemas:
         Draft4Validator.check_schema(schema)
+        ensure_additional_property_is_boolean(schema)
 
 
 def test_spec():

diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md
@@ -94,6 +94,7 @@ Information about expected report generation waiting time you may find [here](ht
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                         |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
+| 1.0.6   | 2023-05-09 | [25913](https://github.com/airbytehq/airbyte/pull/25913) | Small schema fixes                                                                                              |
 | 1.0.5   | 2023-05-08 | [25885](https://github.com/airbytehq/airbyte/pull/25885) | Improve error handling for attribution_report(s) streams                                                        |
 | 1.0.4   | 2023-05-04 | [25792](https://github.com/airbytehq/airbyte/pull/25792) | Add availability strategy for basic streams (not including report streams)                                      |
 | 1.0.3   | 2023-04-13 | [25146](https://github.com/airbytehq/airbyte/pull/25146) | Validate pk for reports when expected pk is not returned                                                        |