🐛 Source Recharge: data types should be transformed based on schema t…

…ypes (#16959)
airbytehq · Sep 22, 2022 · ea3d57b · ea3d57b
1 parent 596a436
commit ea3d57b
Show file tree

Hide file tree

Showing 11 changed files with 25 additions and 125 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -853,7 +853,7 @@
 - name: Recharge
   sourceDefinitionId: 45d2e135-2ede-49e1-939f-3e3ec357a65e
   dockerRepository: airbyte/source-recharge
-  dockerImageTag: 0.1.8
+  dockerImageTag: 0.2.0
   documentationUrl: https://docs.airbyte.io/integrations/sources/recharge
   icon: recharge.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -8772,7 +8772,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-recharge:0.1.8"
+- dockerImage: "airbyte/source-recharge:0.2.0"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/recharge"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-recharge/Dockerfile b/airbyte-integrations/connectors/source-recharge/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.9
+LABEL io.airbyte.version=0.2.0
 LABEL io.airbyte.name=airbyte/source-recharge
diff --git a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml
@@ -8,7 +8,11 @@ tests:
     - config_path: "integration_tests/invalid_config.json"
       status: "failed"
   discovery:
+    # schemas were changed in `0.1.9` compare to `0.1.8`,
+    # plase remove this bypass, once updated to the newer version!
     - config_path: "secrets/config.json"
+      backward_compatibility_tests_config:
+        disable_for_version: "0.1.8"
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/streams_with_output_records_catalog.json"

diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py
@@ -20,6 +20,9 @@ class RechargeStream(HttpStream, ABC):
     limit = 250
     page_num = 1
 
+    # regestring the default schema transformation
+    transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization)
+
     @property
     def data_path(self):
         return self.name
@@ -109,20 +112,6 @@ class Charges(IncrementalRechargeStream):
     Charges Stream: https://developer.rechargepayments.com/v1-shopify?python#list-charges
     """
 
-    def get_stream_data(self, response_data: Any) -> List[dict]:
-        # We expect total_weight to be an integer, but the API is returning numbers like 42.0
-        # Cast these down to int if possible, and error if not.
-        data = super().get_stream_data(response_data)
-        for record in data:
-            if "total_weight" in record:
-                total_weight = record["total_weight"]
-                int_total_weight = int(total_weight)
-                if total_weight == int_total_weight:
-                    record["total_weight"] = int_total_weight
-                else:
-                    raise ValueError(f"Expected total_weight to be an integer, got {total_weight}")
-        return data
-
 
 class Collections(RechargeStream):
     """
@@ -164,8 +153,6 @@ class Orders(IncrementalRechargeStream):
     Orders Stream: https://developer.rechargepayments.com/v1-shopify?python#list-orders
     """
 
-    transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization)
-
 
 class Products(RechargeStream):
     """

diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json
@@ -226,34 +226,13 @@
       "type": ["null", "string"]
     },
     "tax_lines": {
-      "oneOf": [
-        {
-          "type": ["null", "number"]
-        },
-        {
-          "type": ["null", "string"]
-        }
-      ]
+      "type": ["null", "string"]
     },
     "total_discounts": {
-      "oneOf": [
-        {
-          "type": ["null", "number"]
-        },
-        {
-          "type": ["null", "string"]
-        }
-      ]
+      "type": ["null", "string"]
     },
     "total_line_items_price": {
-      "oneOf": [
-        {
-          "type": ["null", "number"]
-        },
-        {
-          "type": ["null", "string"]
-        }
-      ]
+      "type": ["null", "string"]
     },
     "total_price": {
       "type": ["null", "string"]
@@ -262,17 +241,10 @@
       "type": ["null", "string"]
     },
     "total_tax": {
-      "oneOf": [
-        {
-          "type": ["null", "number"]
-        },
-        {
-          "type": ["null", "string"]
-        }
-      ]
+      "type": ["null", "string"]
     },
     "total_weight": {
-      "type": ["null", "integer"]
+      "type": ["null", "number"]
     },
     "transaction_id": {
       "type": ["null", "string"]

diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/metafields.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/metafields.json
@@ -19,14 +19,7 @@
       "type": ["null", "string"]
     },
     "owner_id": {
-      "oneOf": [
-        {
-          "type": ["null", "number"]
-        },
-        {
-          "type": ["null", "string"]
-        }
-      ]
+      "type": ["null", "string"]
     },
     "owner_resource": {
       "type": ["null", "string"]
@@ -36,14 +29,7 @@
       "format": "date-time"
     },
     "value": {
-      "oneOf": [
-        {
-          "type": ["null", "number"]
-        },
-        {
-          "type": ["null", "string"]
-        }
-      ]
+      "type": ["null", "string"]
     },
     "value_type": {
       "type": ["null", "string"]

diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/onetimes.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/onetimes.json
@@ -6,42 +6,21 @@
       "type": ["null", "integer"]
     },
     "address_id": {
-      "oneOf": [
-        {
-          "type": ["null", "number"]
-        },
-        {
-          "type": ["null", "string"]
-        }
-      ]
+      "type": ["null", "string"]
     },
     "created_at": {
       "type": ["null", "string"],
       "format": "date-time"
     },
     "customer_id": {
-      "oneOf": [
-        {
-          "type": ["null", "number"]
-        },
-        {
-          "type": ["null", "string"]
-        }
-      ]
+      "type": ["null", "string"]
     },
     "next_charge_scheduled_at": {
       "type": ["null", "string"],
       "format": "date-time"
     },
     "price": {
-      "oneOf": [
-        {
-          "type": ["null", "number"]
-        },
-        {
-          "type": ["null", "string"]
-        }
-      ]
+      "type": ["null", "string"]
     },
     "product_title": {
       "type": ["null", "string"]

diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/orders.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/orders.json
@@ -219,24 +219,10 @@
             "type": ["null", "integer"]
           },
           "shopify_product_id": {
-            "oneOf": [
-              {
-                "type": ["null", "number"]
-              },
-              {
-                "type": ["null", "string"]
-              }
-            ]
+            "type": ["null", "string"]
           },
           "shopify_variant_id": {
-            "oneOf": [
-              {
-                "type": ["null", "number"]
-              },
-              {
-                "type": ["null", "string"]
-              }
-            ]
+            "type": ["null", "string"]
           },
           "sku": {
             "type": ["null", "string"]
@@ -381,14 +367,7 @@
       "type": ["null", "string"]
     },
     "total_tax": {
-      "oneOf": [
-        {
-          "type": ["null", "number"]
-        },
-        {
-          "type": ["null", "string"]
-        }
-      ]
+      "type": ["null", "string"]
     },
     "total_weight": {
       "type": ["null", "integer"]

diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/subscriptions.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/subscriptions.json
@@ -85,14 +85,7 @@
             "type": "string"
           },
           "value": {
-            "oneOf": [
-              {
-                "type": ["null", "number"]
-              },
-              {
-                "type": ["null", "string"]
-              }
-            ]
+            "type": ["null", "string"]
           }
         }
       }

diff --git a/docs/integrations/sources/recharge.md b/docs/integrations/sources/recharge.md
@@ -67,7 +67,7 @@ The Recharge connector should gracefully handle Recharge API limitations under n
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
-| 0.1.9 | 2022-09-12 | [16592](https://github.com/airbytehq/airbyte/pull/16592) | Use TypeTransformer to reliably convert to int values in Orders stream
+| 0.2.0 | 2022-09-21 | [16959](https://github.com/airbytehq/airbyte/pull/16959) | Use TypeTransformer to reliably convert to schema declared data types
 | 0.1.8 | 2022-08-27 | [16045](https://github.com/airbytehq/airbyte/pull/16045) | Force total_weight to be an integer |
 | 0.1.7 | 2022-07-24 | [14978](https://github.com/airbytehq/airbyte/pull/14978) | Set `additionalProperties` to True, to guarantee backward cababilities |
 | 0.1.6 | 2022-07-21 | [14902](https://github.com/airbytehq/airbyte/pull/14902) | Increased test coverage, fixed broken `charges`, `orders` schemas, added state checkpoint |