From ea3d57b6f3f7abd4f3caf3546d265fba3a840692 Mon Sep 17 00:00:00 2001 From: Baz Date: Thu, 22 Sep 2022 15:21:21 +0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Source=20Recharge:=20data=20type?= =?UTF-8?q?s=20should=20be=20transformed=20based=20on=20schema=20types=20(?= =?UTF-8?q?#16959)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-recharge/Dockerfile | 2 +- .../acceptance-test-config.yml | 4 ++ .../source-recharge/source_recharge/api.py | 19 ++-------- .../source_recharge/schemas/charges.json | 38 +++---------------- .../source_recharge/schemas/metafields.json | 18 +-------- .../source_recharge/schemas/onetimes.json | 27 ++----------- .../source_recharge/schemas/orders.json | 27 ++----------- .../schemas/subscriptions.json | 9 +---- docs/integrations/sources/recharge.md | 2 +- 11 files changed, 25 insertions(+), 125 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 22a30733a9356..0be765b535b5b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -853,7 +853,7 @@ - name: Recharge sourceDefinitionId: 45d2e135-2ede-49e1-939f-3e3ec357a65e dockerRepository: airbyte/source-recharge - dockerImageTag: 0.1.8 + dockerImageTag: 0.2.0 documentationUrl: https://docs.airbyte.io/integrations/sources/recharge icon: recharge.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 61a86a086a7c0..967c0eaf3f898 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -8772,7 +8772,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-recharge:0.1.8" +- dockerImage: "airbyte/source-recharge:0.2.0" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/recharge" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-recharge/Dockerfile b/airbyte-integrations/connectors/source-recharge/Dockerfile index 5d26ae27191aa..362280b78fca6 100644 --- a/airbyte-integrations/connectors/source-recharge/Dockerfile +++ b/airbyte-integrations/connectors/source-recharge/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.9 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-recharge diff --git a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml index 89a8dd0ce2df3..f6bca0d4336a0 100644 --- a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml @@ -8,7 +8,11 @@ tests: - config_path: "integration_tests/invalid_config.json" status: "failed" discovery: + # schemas were changed in `0.1.9` compare to `0.1.8`, + # plase remove this bypass, once updated to the newer version! - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.8" basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/streams_with_output_records_catalog.json" diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py index acd24fb608a7a..c3350378e427c 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py @@ -20,6 +20,9 @@ class RechargeStream(HttpStream, ABC): limit = 250 page_num = 1 + # regestring the default schema transformation + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + @property def data_path(self): return self.name @@ -109,20 +112,6 @@ class Charges(IncrementalRechargeStream): Charges Stream: https://developer.rechargepayments.com/v1-shopify?python#list-charges """ - def get_stream_data(self, response_data: Any) -> List[dict]: - # We expect total_weight to be an integer, but the API is returning numbers like 42.0 - # Cast these down to int if possible, and error if not. - data = super().get_stream_data(response_data) - for record in data: - if "total_weight" in record: - total_weight = record["total_weight"] - int_total_weight = int(total_weight) - if total_weight == int_total_weight: - record["total_weight"] = int_total_weight - else: - raise ValueError(f"Expected total_weight to be an integer, got {total_weight}") - return data - class Collections(RechargeStream): """ @@ -164,8 +153,6 @@ class Orders(IncrementalRechargeStream): Orders Stream: https://developer.rechargepayments.com/v1-shopify?python#list-orders """ - transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - class Products(RechargeStream): """ diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json index 13df3e38b1042..9c5e62dcdbb18 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json @@ -226,34 +226,13 @@ "type": ["null", "string"] }, "tax_lines": { - "oneOf": [ - { - "type": ["null", "number"] - }, - { - "type": ["null", "string"] - } - ] + "type": ["null", "string"] }, "total_discounts": { - "oneOf": [ - { - "type": ["null", "number"] - }, - { - "type": ["null", "string"] - } - ] + "type": ["null", "string"] }, "total_line_items_price": { - "oneOf": [ - { - "type": ["null", "number"] - }, - { - "type": ["null", "string"] - } - ] + "type": ["null", "string"] }, "total_price": { "type": ["null", "string"] @@ -262,17 +241,10 @@ "type": ["null", "string"] }, "total_tax": { - "oneOf": [ - { - "type": ["null", "number"] - }, - { - "type": ["null", "string"] - } - ] + "type": ["null", "string"] }, "total_weight": { - "type": ["null", "integer"] + "type": ["null", "number"] }, "transaction_id": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/metafields.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/metafields.json index 42c09d4de7b40..228bff45ca171 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/metafields.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/metafields.json @@ -19,14 +19,7 @@ "type": ["null", "string"] }, "owner_id": { - "oneOf": [ - { - "type": ["null", "number"] - }, - { - "type": ["null", "string"] - } - ] + "type": ["null", "string"] }, "owner_resource": { "type": ["null", "string"] @@ -36,14 +29,7 @@ "format": "date-time" }, "value": { - "oneOf": [ - { - "type": ["null", "number"] - }, - { - "type": ["null", "string"] - } - ] + "type": ["null", "string"] }, "value_type": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/onetimes.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/onetimes.json index e010e4274f356..9263fa07ddbc2 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/onetimes.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/onetimes.json @@ -6,42 +6,21 @@ "type": ["null", "integer"] }, "address_id": { - "oneOf": [ - { - "type": ["null", "number"] - }, - { - "type": ["null", "string"] - } - ] + "type": ["null", "string"] }, "created_at": { "type": ["null", "string"], "format": "date-time" }, "customer_id": { - "oneOf": [ - { - "type": ["null", "number"] - }, - { - "type": ["null", "string"] - } - ] + "type": ["null", "string"] }, "next_charge_scheduled_at": { "type": ["null", "string"], "format": "date-time" }, "price": { - "oneOf": [ - { - "type": ["null", "number"] - }, - { - "type": ["null", "string"] - } - ] + "type": ["null", "string"] }, "product_title": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/orders.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/orders.json index 5357ba81883f0..26546ab2f554a 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/orders.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/orders.json @@ -219,24 +219,10 @@ "type": ["null", "integer"] }, "shopify_product_id": { - "oneOf": [ - { - "type": ["null", "number"] - }, - { - "type": ["null", "string"] - } - ] + "type": ["null", "string"] }, "shopify_variant_id": { - "oneOf": [ - { - "type": ["null", "number"] - }, - { - "type": ["null", "string"] - } - ] + "type": ["null", "string"] }, "sku": { "type": ["null", "string"] @@ -381,14 +367,7 @@ "type": ["null", "string"] }, "total_tax": { - "oneOf": [ - { - "type": ["null", "number"] - }, - { - "type": ["null", "string"] - } - ] + "type": ["null", "string"] }, "total_weight": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/subscriptions.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/subscriptions.json index 7f1b0a9bcd7d2..7556bb211549d 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/subscriptions.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/subscriptions.json @@ -85,14 +85,7 @@ "type": "string" }, "value": { - "oneOf": [ - { - "type": ["null", "number"] - }, - { - "type": ["null", "string"] - } - ] + "type": ["null", "string"] } } } diff --git a/docs/integrations/sources/recharge.md b/docs/integrations/sources/recharge.md index 638369987a7f8..946194256c5dc 100644 --- a/docs/integrations/sources/recharge.md +++ b/docs/integrations/sources/recharge.md @@ -67,7 +67,7 @@ The Recharge connector should gracefully handle Recharge API limitations under n | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.9 | 2022-09-12 | [16592](https://github.com/airbytehq/airbyte/pull/16592) | Use TypeTransformer to reliably convert to int values in Orders stream +| 0.2.0 | 2022-09-21 | [16959](https://github.com/airbytehq/airbyte/pull/16959) | Use TypeTransformer to reliably convert to schema declared data types | 0.1.8 | 2022-08-27 | [16045](https://github.com/airbytehq/airbyte/pull/16045) | Force total_weight to be an integer | | 0.1.7 | 2022-07-24 | [14978](https://github.com/airbytehq/airbyte/pull/14978) | Set `additionalProperties` to True, to guarantee backward cababilities | | 0.1.6 | 2022-07-21 | [14902](https://github.com/airbytehq/airbyte/pull/14902) | Increased test coverage, fixed broken `charges`, `orders` schemas, added state checkpoint |