🎉 Source Amazon Seller Partner: enable stream attribute customisation…

… from Source configuration (#22503) * feat: add new field and fix tests to accept it * renamed to advanced_stream_options * add implementation with test * feat: add new field and fix tests to accept it * renamed to advanced_stream_options * add implementation with test * update specs * fix lint errors * increment Dockerfile version * Add missing field for spec validation * add doc and update mtdt --------- Co-authored-by: sh4sh <6833405+sh4sh@users.noreply.github.com> Co-authored-by: Marcos Marx <marcosmarxm@users.noreply.github.com> Co-authored-by: Sajarin <sajarindider@gmail.com> Co-authored-by: marcosmarxm <marcosmarxm@gmail.com>
airbytehq · May 23, 2023 · 4e86231 · 4e86231
1 parent 53d66be
commit 4e86231
Show file tree

Hide file tree

Showing 12 changed files with 76 additions and 2 deletions.
diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml
@@ -4,7 +4,7 @@ acceptance_tests:
     tests:
       - spec_path: "integration_tests/spec.json"
         backward_compatibility_tests_config:
-          disable_for_version: "0.2.33"
+          disable_for_version: "1.1.0"
   connection:
     tests:
       - config_path: "secrets/config.json"

diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec.json
@@ -102,6 +102,16 @@
         "order": 12,
         "type": "integer"
       },
+      "advanced_stream_options": {
+        "title": "Advanced Stream Options",
+        "description": "Additional information to configure report options. This varies by report type, not every report implement this kind of feature. Must be a valid json string.",
+        "examples": [
+          "{\"GET_SALES_AND_TRAFFIC_REPORT\": {\"availability_sla_days\": 3}}",
+          "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}"
+        ],
+        "order": 13,
+        "type": "string"
+      },
       "aws_environment": {
         "title": "AWSEnvironment",
         "description": "An enumeration.",

diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml
@@ -2,7 +2,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460
-  dockerImageTag: 1.1.0
+  dockerImageTag: 1.2.0
   dockerRepository: airbyte/source-amazon-seller-partner
   githubIssueLabel: source-amazon-seller-partner
   icon: amazonsellerpartner.svg

diff --git a/...tegrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py b/...tegrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py
@@ -98,6 +98,7 @@ def _get_stream_kwargs(self, config: AmazonSellerPartnerConfig) -> Mapping[str,
             "report_options": config.report_options,
             "max_wait_seconds": config.max_wait_seconds,
             "replication_end_date": config.replication_end_date,
+            "advanced_stream_options": config.advanced_stream_options,
         }
         return stream_kwargs
 

diff --git a/...integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.py b/...integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.py
@@ -108,6 +108,16 @@ class Config:
         order=12,
     )
 
+    # This field has been introduced because some users experienced a delay on report data update,
+    # for certain reports the data availability SLA is not explicited this field allows to customize this value. Using this sort of
+    # options make possible to extend the usage adding more customization per stream.
+    advanced_stream_options: str = Field(
+        None,
+        description="Additional information to configure report options. This varies by report type, not every report implement this kind of feature. Must be a valid json string.",
+        examples=['{"GET_SALES_AND_TRAFFIC_REPORT": {"availability_sla_days": 3}}', '{"GET_SOME_REPORT": {"custom": "true"}}'],
+        order=13,
+    )
+
     aws_environment: AWSEnvironment = Field(
         description="Select the AWS Environment.",
         title="AWS Environment",

diff --git a/...egrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/...egrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py
@@ -44,6 +44,7 @@ def __init__(
         marketplace_id: str,
         period_in_days: Optional[int],
         report_options: Optional[str],
+        advanced_stream_options: Optional[str],
         max_wait_seconds: Optional[int],
         replication_end_date: Optional[str],
         *args,
@@ -173,6 +174,7 @@ def __init__(
         report_options: Optional[str],
         max_wait_seconds: Optional[int],
         replication_end_date: Optional[str],
+        advanced_stream_options: Optional[str],
         authenticator: HttpAuthenticator = None,
     ):
         self._authenticator = authenticator
@@ -185,6 +187,9 @@ def __init__(
         self.period_in_days = max(period_in_days, self.replication_start_date_limit_in_days)  # ensure old configs work as well
         self._report_options = report_options or "{}"
         self.max_wait_seconds = max_wait_seconds
+        self._advanced_stream_options = dict()
+        if advanced_stream_options is not None:
+            self._advanced_stream_options = json_lib.loads(advanced_stream_options)
 
     @property
     def url_base(self) -> str:
@@ -915,6 +920,15 @@ class SellerAnalyticsSalesAndTrafficReports(IncrementalAnalyticsStream):
     cursor_field = "queryEndDate"
     fixed_period_in_days = 1
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        if self.name in self._advanced_stream_options.keys():
+            _options: dict = self._advanced_stream_options[self.name]
+            if isinstance(_options, dict):
+                for _option_attr, _option_val in _options.items():
+                    setattr(self, _option_attr, _option_val)
+
 
 class VendorSalesReports(IncrementalAnalyticsStream):
     name = "GET_VENDOR_SALES_REPORT"

diff --git a/...e-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py b/...e-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py
@@ -112,6 +112,7 @@ def _internal(start_date: str = START_DATE_1, end_date: str = END_DATE_1):
             authenticator=None,
             period_in_days=0,
             report_options=None,
+            advanced_stream_options=None,
             max_wait_seconds=500,
         )
         return stream
@@ -138,6 +139,7 @@ def _internal(start_date: str = START_DATE_1, end_date: str = END_DATE_1):
             authenticator=None,
             period_in_days=0,
             report_options=None,
+            advanced_stream_options=None,
             max_wait_seconds=500,
         )
         return stream

diff --git a/...nnectors/source-amazon-seller-partner/unit_tests/test_reports_stream_sales_and_traffic.py b/...nnectors/source-amazon-seller-partner/unit_tests/test_reports_stream_sales_and_traffic.py
@@ -0,0 +1,33 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from source_amazon_seller_partner.auth import AWSSignature
+from source_amazon_seller_partner.streams import SellerAnalyticsSalesAndTrafficReports
+
+START_DATE_1 = "2023-02-05T00:00:00Z"
+END_DATE_1 = "2023-02-07T00:00:00Z"
+
+
+def test_stream_uses_advanced_options():
+    aws_signature = AWSSignature(
+            service="execute-api",
+            aws_access_key_id="AccessKeyId",
+            aws_secret_access_key="SecretAccessKey",
+            aws_session_token="SessionToken",
+            region="US",
+        )
+    stream = SellerAnalyticsSalesAndTrafficReports(
+            url_base="https://test.url",
+            aws_signature=aws_signature,
+            replication_start_date=START_DATE_1,
+            replication_end_date=END_DATE_1,
+            marketplace_id="id",
+            authenticator=None,
+            period_in_days=0,
+            report_options=None,
+            advanced_stream_options='{"GET_SALES_AND_TRAFFIC_REPORT":{"availability_sla_days": 3}}',
+            max_wait_seconds=500,
+        )
+
+    assert stream.availability_sla_days == 3
diff --git a/...ns/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_rate_limits.py b/...ns/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_rate_limits.py
@@ -27,6 +27,7 @@ def reports_stream():
         authenticator=None,
         period_in_days=0,
         report_options=None,
+        advanced_stream_options=None,
         max_wait_seconds=500,
     )
     return stream

diff --git a/...nectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py b/...nectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py
@@ -94,6 +94,7 @@ def _internal(start_date: str = START_DATE_1, end_date: str = END_DATE_1):
             authenticator=None,
             period_in_days=0,
             report_options=None,
+            advanced_stream_options=None,
             max_wait_seconds=500,
         )
         return stream

diff --git a/...ntegrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py b/...ntegrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py
@@ -24,6 +24,7 @@ def reports_stream(marketplace_id):
         authenticator=None,
         period_in_days=0,
         report_options=None,
+        advanced_stream_options=None,
         max_wait_seconds=0,
     )
     return stream

diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md
@@ -124,6 +124,7 @@ So, for any value that exceeds the limit, the `period_in_days` will be automatic
 
 | Version  | Date       | Pull Request                                               | Subject                                                                                                                                                                             |
 |:---------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `1.2.0`  | 2023-05-23 | [\#22503](https://github.com/airbytehq/airbyte/pull/22503) | Enabled stream attribute customization from Source configuration                      |
 | `1.1.0`  | 2023-04-21 | [\#23605](https://github.com/airbytehq/airbyte/pull/23605) | Add FBA Reimbursement Report stream                                    |
 | `1.0.1`  | 2023-03-15 | [\#24098](https://github.com/airbytehq/airbyte/pull/24098) | Add Belgium Marketplace                                                                                                                                                             |
 | `1.0.0`  | 2023-03-13 | [\#23980](https://github.com/airbytehq/airbyte/pull/23980) | Make `app_id` required. Increase `end_date` gap up to 5 minutes from now for Finance streams. Fix connection check failure when trying to connect to Amazon Vendor Central accounts |