Source Amazon Ads: fix Nonetype error when recordId is missing (airby…

…tehq#28155) * Source Amazon Ads: fix Nonetype error when recordId is missing * Source Amazon Ads: bump version and update changelog * Source Amazon Ads: unittest for recordId generation
scentbird · Jul 27, 2023 · 7ca3512 · 7ca3512
1 parent e9b493c
commit 7ca3512
Show file tree

Hide file tree

Showing 5 changed files with 30 additions and 4 deletions.
diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile
@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=2.3.0
+LABEL io.airbyte.version=2.3.1
 LABEL io.airbyte.name=airbyte/source-amazon-ads
diff --git a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml
@@ -8,7 +8,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
-  dockerImageTag: 2.3.0
+  dockerImageTag: 2.3.1
   dockerRepository: airbyte/source-amazon-ads
   githubIssueLabel: source-amazon-ads
   icon: amazonads.svg

diff --git a/...s/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/...s/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py
@@ -171,10 +171,13 @@ def read_records(
                     profileId=report_info.profile_id,
                     recordType=report_info.record_type,
                     reportDate=report_date,
-                    recordId=metric_object.get(self.metrics_type_to_id_map[report_info.record_type], str(uuid.uuid4())),
+                    recordId=self.get_record_id(metric_object, report_info.record_type),
                     metric=metric_object,
                 ).dict()
 
+    def get_record_id(self, metric_object: dict, record_type: str) -> str:
+        return metric_object.get(self.metrics_type_to_id_map[record_type]) or str(uuid.uuid4())
+
     def backoff_max_time(func):
         def wrapped(self, *args, **kwargs):
             return backoff.on_exception(backoff.constant, RetryableException, max_time=self.report_wait_timeout * 60, interval=10)(func)(

diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py
@@ -898,3 +898,25 @@ def test_brands_video_report_with_custom_record_types(config_gen, custom_record_
         if record['recordType'] not in expected_record_types:
             if flag_match_error:
                 assert False
+
+
+@pytest.mark.parametrize(
+    "metric_object, record_type",
+    [
+        ({"campaignId": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}, "campaigns"),
+        ({"campaignId": ""}, "campaigns"),
+        ({"campaignId": None}, "campaigns")
+    ]
+)
+def test_get_record_id_by_report_type(config, metric_object, record_type):
+    """
+    Test if a `recordId` is allways non-empty for any given `metric_object`.
+    `recordId` is not a contant key for every report.
+    We define suitable key for every report by its type and normally it should not be empty.
+    It may be `campaignId` or `adGroupId` or any other key depending on report type (See METRICS_TYPE_TO_ID_MAP).
+    In case when it is not defined or empty (sometimes we get one record with missing data while others are populated)
+        we must return `recordId` anyway so we generate it manually.
+    """
+    profiles = make_profiles(profile_type="vendor")
+    stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock())
+    assert stream.get_record_id(metric_object, record_type), "recordId must be non-empty value"
diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md
@@ -99,7 +99,8 @@ Information about expected report generation waiting time you may find [here](ht
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                         |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| 2.3.0   | 2023-07-06 | [28002](https://github.com/airbytehq/airbyte/pull/28002) | Add sponsored_product_ad_group_suggested_keywords, sponsored_product_ad_group_bid_recommendations streams                        |
+| 2.3.1   | 2023-07-11 | [28155](https://github.com/airbytehq/airbyte/pull/28155) | Bugfix: validation error when record values are missing                                                         |
+| 2.3.0   | 2023-07-06 | [28002](https://github.com/airbytehq/airbyte/pull/28002) | Add sponsored_product_ad_group_suggested_keywords, sponsored_product_ad_group_bid_recommendations streams       |
 | 2.2.0   | 2023-07-05 | [27607](https://github.com/airbytehq/airbyte/pull/27607) | Add stream for sponsored brands v3 purchased product reports                                                    |
 | 2.1.0   | 2023-06-19 | [25412](https://github.com/airbytehq/airbyte/pull/25412) | Add sponsored_product_campaign_negative_keywords, sponsored_display_budget_rules streams                        |
 | 2.0.0   | 2023-05-31 | [25874](https://github.com/airbytehq/airbyte/pull/25874) | Type `portfolioId` as integer                                                                                   |