Source Amazon Seller Partner: extract REPORTS_MAX_WAIT_SECONDS to con…

…figurable parameter (#9002)

* Source Amazon Seller Partner: extract REPORTS_MAX_WAIT_SECONDS to configurable parameter

* docs

* 9002

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e55879a8-0ef8-4557-abcf-ab34c53ec460.json

@@ -2,7 +2,7 @@
   "sourceDefinitionId": "e55879a8-0ef8-4557-abcf-ab34c53ec460",
   "name": "Amazon Seller Partner",
   "dockerRepository": "airbyte/source-amazon-seller-partner",
-  "dockerImageTag": "0.2.6",
+  "dockerImageTag": "0.2.7",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/amazon-seller-partner",
   "icon": "amazonsellerpartner.svg"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -22,7 +22,7 @@
 - name: Amazon Seller Partner
   sourceDefinitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460
   dockerRepository: airbyte/source-amazon-seller-partner
-  dockerImageTag: 0.2.6
+  dockerImageTag: 0.2.7
   sourceType: api
   documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-seller-partner
   icon: amazonsellerpartner.svg

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -153,7 +153,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-amazon-seller-partner:0.2.6"
+- dockerImage: "airbyte/source-amazon-seller-partner:0.2.7"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/amazon-seller-partner"
     changelogUrl: "https://docs.airbyte.io/integrations/sources/amazon-seller-partner"

airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.6
+LABEL io.airbyte.version=0.2.7
 LABEL io.airbyte.name=airbyte/source-amazon-seller-partner

airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py

@@ -50,6 +50,12 @@ class Config:
         description="Additional information passed to reports. This varies by report type. Must be a valid json string.",
         examples=['{"GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT": {"reportPeriod": "WEEK"}}', '{"GET_SOME_REPORT": {"custom": "true"}}'],
     )
+    max_wait_seconds: int = Field(
+        500,
+        title="Max wait time for reports (in seconds)",
+        description="Sometimes report can take up to 30 minutes to generate. This will set the limit for how long to wait for a successful report.",
+        examples=["500", "1980"],
+    )
     refresh_token: str = Field(
         description="The Refresh Token obtained via OAuth flow authorization.",
         title="Refresh Token",
@@ -105,6 +111,7 @@ def _get_stream_kwargs(self, config: ConnectorConfig) -> Mapping[str, Any]:
             "marketplace_ids": [marketplace_id],
             "period_in_days": config.period_in_days,
             "report_options": config.report_options,
+            "max_wait_seconds": config.max_wait_seconds,
         }
         return stream_kwargs
 

airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json

@@ -28,6 +28,13 @@
         ],
         "type": "string"
       },
+      "max_wait_seconds": {
+        "title": "Max wait time for reports (in seconds)",
+        "description": "Sometimes report can take up to 30 minutes to generate. This will set the limit for how long to wait for a successful report.",
+        "examples": ["500", "1980"],
+        "type": "integer",
+        "default": 500
+      },
       "refresh_token": {
         "title": "Refresh Token",
         "description": "The Refresh Token obtained via OAuth flow authorization.",

airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py

@@ -30,9 +30,6 @@
 ORDERS_API_VERSION = "v0"
 VENDORS_API_VERSION = "v1"
 
-# 33min. taken from real world experience working with amazon seller partner reports
-REPORTS_MAX_WAIT_SECONDS = 1980
-
 DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
 
 
@@ -47,6 +44,7 @@ def __init__(
         marketplace_ids: List[str],
         period_in_days: Optional[int],
         report_options: Optional[str],
+        max_wait_seconds: Optional[int],
         *args,
         **kwargs,
     ):
@@ -153,6 +151,7 @@ def __init__(
         marketplace_ids: List[str],
         period_in_days: Optional[int],
         report_options: Optional[str],
+        max_wait_seconds: Optional[int],
         authenticator: HttpAuthenticator = NoAuth(),
     ):
         self._authenticator = authenticator
@@ -163,6 +162,7 @@ def __init__(
         self.marketplace_ids = marketplace_ids
         self.period_in_days = period_in_days
         self._report_options = report_options
+        self.max_wait_seconds = max_wait_seconds
 
     @property
     def url_base(self) -> str:
@@ -314,7 +314,7 @@ def read_records(
         report_id = self._create_report(sync_mode, cursor_field, stream_slice, stream_state)["reportId"]
 
         # create and retrieve the report
-        while not is_processed and seconds_waited < REPORTS_MAX_WAIT_SECONDS:
+        while not is_processed and seconds_waited < self.max_wait_seconds:
             report_payload = self._retrieve_report(report_id=report_id)
             seconds_waited = (pendulum.now("utc") - start_time).seconds
             is_processed = report_payload.get("processingStatus") not in ["IN_QUEUE", "IN_PROGRESS"]

airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_repots_streams_rate_limits.py

@@ -29,6 +29,7 @@ def reports_stream():
         authenticator=NoAuth(),
         period_in_days=0,
         report_options=None,
+        max_wait_seconds=500,
     )
     return stream
 

docs/integrations/sources/amazon-seller-partner.md

@@ -64,6 +64,7 @@ Information about rate limits you may find [here](https://github.com/amzn/sellin
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| `0.2.7` | 2021-12-21 | [\#9002](https://github.com/airbytehq/airbyte/pull/9002) | Extract REPORTS_MAX_WAIT_SECONDS to configurable parameter |
 | `0.2.6` | 2021-12-10 | [\#8179](https://github.com/airbytehq/airbyte/pull/8179) | Add GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT report |
 | `0.2.5` | 2021-12-06 | [\#8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec |
 | `0.2.4` | 2021-11-08 | [\#8021](https://github.com/airbytehq/airbyte/pull/8021) | Added GET_SELLER_FEEDBACK_DATA report with incremental sync capability |