🎉 Source Shopify: dynamically adjust the size of the slice for Bulk…

… API streams (#36788)
airbytehq · Apr 17, 2024 · dccb6c0 · dccb6c0
1 parent 0f3ab40
commit dccb6c0
Show file tree

Hide file tree

Showing 14 changed files with 549 additions and 184 deletions.
diff --git a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml
@@ -61,7 +61,7 @@ acceptance_tests:
         configured_catalog_path: "integration_tests/configured_catalog.json"
         future_state:
           future_state_path: "integration_tests/abnormal_state.json"
-        timeout_seconds: 7200
+        timeout_seconds: 8400
   full_refresh:
     tests:
       - config_path: "secrets/config.json"

diff --git a/airbyte-integrations/connectors/source-shopify/metadata.yaml b/airbyte-integrations/connectors/source-shopify/metadata.yaml
@@ -11,13 +11,13 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 9da77001-af33-4bcd-be46-6252bf9342b9
-  dockerImageTag: 2.0.4
+  dockerImageTag: 2.0.5
   dockerRepository: airbyte/source-shopify
   documentationUrl: https://docs.airbyte.com/integrations/sources/shopify
   githubIssueLabel: source-shopify
   icon: shopify.svg
   license: ELv2
-  maxSecondsBetweenMessages: 7200
+  maxSecondsBetweenMessages: 21600
   name: Shopify
   remoteRegistries:
     pypi:

diff --git a/airbyte-integrations/connectors/source-shopify/poetry.lock b/airbyte-integrations/connectors/source-shopify/poetry.lock
diff --git a/airbyte-integrations/connectors/source-shopify/pyproject.toml b/airbyte-integrations/connectors/source-shopify/pyproject.toml
@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "2.0.4"
+version = "2.0.5"
 name = "source-shopify"
 description = "Source CDK implementation for Shopify."
 authors = [ "Airbyte <contact@airbyte.io>",]
@@ -17,7 +17,7 @@ include = "source_shopify"
 
 [tool.poetry.dependencies]
 python = "^3.9,<3.12"
-airbyte-cdk = ">=0.73.0"
+airbyte-cdk = "^0"
 sgqlc = "==16.3"
 graphql-query = "^1.1.1"
 

diff --git a/...-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/exceptions.py b/...-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/exceptions.py
@@ -9,8 +9,12 @@
 
 class ShopifyBulkExceptions:
     class BaseBulkException(AirbyteTracedException):
+        """Base BULK Job Exception"""
+
+        failure_type: FailureType = FailureType.config_error
+
         def __init__(self, message: str, **kwargs) -> None:
-            super().__init__(internal_message=message, failure_type=FailureType.config_error, **kwargs)
+            super().__init__(internal_message=message, failure_type=self.failure_type, **kwargs)
 
     class BulkJobError(BaseBulkException):
         """Raised when there are BULK Job Errors in response"""
@@ -30,6 +34,11 @@ class BulkRecordProduceError(BaseBulkException):
     class BulkJobFailed(BaseBulkException):
         """Raised when BULK Job has FAILED status"""
 
+    class BulkJobCanceled(BaseBulkException):
+        """Raised when BULK Job has CANCELED status"""
+
+        failure_type: FailureType = FailureType.system_error
+
     class BulkJobTimout(BaseBulkException):
         """Raised when BULK Job has TIMEOUT status"""