🐛 Source Recharge: migrated to 2021-11 api version (#27503)

* migrated to 2021-11 api version * 🤖 Auto format source-recharge code [skip ci] * updated changelog * updated request params * increased timeout for sat * increased timeout for sat * removed stream slices * added stream_slices --------- Co-authored-by: octavia-squidington-iii <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Jun 26, 2023 · c71b8f8 · c71b8f8
1 parent 2f02d7f
commit c71b8f8
Show file tree

Hide file tree

Showing 10 changed files with 94 additions and 91 deletions.
diff --git a/airbyte-integrations/connectors/source-recharge/Dockerfile b/airbyte-integrations/connectors/source-recharge/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.9
+LABEL io.airbyte.version=0.2.10
 LABEL io.airbyte.name=airbyte/source-recharge
diff --git a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml
@@ -19,7 +19,7 @@ acceptance_tests:
             bypass_reason: "updated after login"
           - name: store/updated_at
             bypass_reason: "updated after login"
-      timeout_seconds: 1200
+      timeout_seconds: 7200
       expect_records:
         path: "integration_tests/expected_records.jsonl"
         extra_fields: no
@@ -41,14 +41,14 @@ acceptance_tests:
     tests:
     - config_path: secrets/config.json
       configured_catalog_path: integration_tests/configured_catalog.json
-      timeout_seconds: 1200
+      timeout_seconds: 3200
   incremental:
     tests:
     - config_path: secrets/config.json
       configured_catalog_path: integration_tests/streams_with_output_records_catalog.json
       future_state:
         future_state_path: integration_tests/abnormal_state.json
-      timeout_seconds: 900
+      timeout_seconds: 3200
   spec:
     tests:
     - spec_path: source_recharge/spec.json

diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/__init__.py b/airbyte-integrations/connectors/source-recharge/integration_tests/__init__.py
@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl
@@ -1,6 +1,6 @@
-{"stream": "addresses", "data": {"address1": "1 9th Ave", "address2": "1", "cart_attributes": null, "cart_note": null, "city": "San Francisco", "company": null, "country": "United States", "country_code": "US", "created_at": "2021-05-12T08:04:06", "customer_id": 64817252, "discount_id": null, "first_name": "Jane", "id": 69105381, "last_name": "Doe", "note_attributes": null, "original_shipping_lines": null, "phone": "1234567890", "presentment_currency": "USD", "province": "California", "shipping_lines_override": null, "updated_at": "2023-01-16T04:59:09", "zip": "94118"}, "emitted_at": 1680895024611}
-{"stream": "charges", "data": {"address_id": 69105381, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country": "United States", "first_name": "Karina", "last_name": "Kuznetsova", "phone": null, "province": "California", "zip": "94118"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2021-05-12T08:04:07", "currency": "USD", "customer_hash": "23dee52d73734a81", "customer_id": 64817252, "discount_codes": [], "email": "nikolaevaka@yahoo.com", "error": "None\r\n [May 12, 12:06AM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 13,  4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 19,  4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 25,  4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 31,  4:09PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [Jun 06,  4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']", "error_type": "CLOSED_MAX_RETRIES_REACHED", "first_name": "Karina", "has_uncommited_changes": false, "id": 386976088, "last_charge_attempt_date": "2022-06-06T16:10:19", "last_name": "Kuznetsova", "line_items": [{"grams": 0, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "price": "24.30", "properties": [], "quantity": 1, "shopify_product_id": "6642695864491", "shopify_variant_id": "39684722131115", "sku": "T1", "subscription_id": 153224593, "tax_lines": [], "title": "Airbit Box Corner Short sleeve t-shirt", "type": "SUBSCRIPTION", "variant_title": "S / Black", "vendor": "Airbyte"}], "note": "", "note_attributes": null, "number_times_tried": 6, "processor_name": "shopify_payments", "requires_shipping": true, "retry_date": "2022-06-12T00:00:00", "scheduled_at": "2022-05-12T00:00:00", "shipments_count": null, "shipping_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country": "United States", "first_name": "Jane", "last_name": "Doe", "phone": "1234567890", "province": "California", "zip": "94118"}, "shipping_lines": [{"code": "Economy", "description": "", "price": 4.9, "source": "shopify", "tax_lines": [], "title": "Economy"}], "shopify_order_id": null, "shopify_variant_id_not_found": null, "status": "ERROR", "sub_total": null, "subtotal_price": "24.3", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "0.0", "total_discounts": "0.00", "total_duties": 0.0, "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": null, "total_tax": "0.0", "total_weight": 0, "transaction_id": null, "type": "RECURRING", "updated_at": "2023-01-16T13:08:54"}, "emitted_at": 1680895025756}
-{"stream": "customers", "data": {"accepts_marketing": 1, "analytics_data": {"utm_params": []}, "billing_address1": "1 9th Ave", "billing_address2": "1", "billing_city": "San Francisco", "billing_company": null, "billing_country": "United States", "billing_phone": null, "billing_province": "California", "billing_zip": "94118", "created_at": "2021-05-12T08:04:06", "email": "nikolaevaka@yahoo.com", "first_charge_processed_at": "2021-05-12T12:03:59", "first_name": "Karina", "has_card_error_in_dunning": false, "has_valid_payment_method": true, "hash": "23dee52d73734a81", "id": 64817252, "last_name": "Kuznetsova", "number_active_subscriptions": 0, "number_subscriptions": 1, "phone": null, "processor_type": "shopify_payments", "reason_payment_method_not_valid": null, "shopify_customer_id": "5212085977259", "status": "ACTIVE", "tax_exempt": false, "updated_at": "2023-01-16T13:08:45"}, "emitted_at": 1680895026982}
+{"stream": "addresses", "data": {"id": 69105381, "customer_id": 64817252, "payment_method_id": 12482012, "address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "created_at": "2021-05-12T12:04:06+00:00", "discounts": [], "first_name": "Jane", "last_name": "Doe", "order_attributes": [], "order_note": null, "phone": "1234567890", "presentment_currency": "USD", "province": "California", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-01-16T09:59:09+00:00", "zip": "94118"}, "emitted_at": 1680895024611}
+{"stream": "charges", "data": {"id": 386976088, "address_id": 69105381, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Karina", "last_name": "Kuznetsova", "phone": null, "province": "California", "zip": "94118"}, "charge_attempts": 6, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2021-05-12T12:04:07+00:00", "currency": "USD", "customer": {"id": 64817252, "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "hash": "23dee52d73734a81"}, "discounts": [], "error": "None\r\n [May 12, 12:06AM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 13,  4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 19,  4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 25,  4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 31,  4:09PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [Jun 06,  4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']", "error_type": "CLOSED_MAX_RETRIES_REACHED", "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "external_variant_not_found": null, "has_uncommitted_changes": false, "last_charge_attempt": "2022-06-06T20:10:19+00:00", "line_items": [{"purchase_item_id": 153224593, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684722131115"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T1", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "S / Black"}], "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": "2022-06-12T04:00:00+00:00", "scheduled_at": "2022-05-12", "shipping_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Jane", "last_name": "Doe", "phone": "1234567890", "province": "California", "zip": "94118"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "source": "shopify", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "error", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-01-16T18:08:54+00:00"}, "emitted_at": 1687184458990}
+{"stream": "customers", "data": {"id": 64817252, "analytics_data": {"utm_params": []}, "created_at": "2021-05-12T12:04:06+00:00", "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "first_charge_processed_at": "2021-05-12T16:03:59+00:00", "first_name": "Karina", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "23dee52d73734a81", "last_name": "Kuznetsova", "phone": null, "subscriptions_active_count": 0, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-01-16T18:08:45+00:00"}, "emitted_at": 1687184599794}
 {"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T07:27:34", "discount_amount": 5.0, "discount_type": "percentage", "handle": "i-make-beats-wool-blend-snapback", "id": 1853639, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_small.jpg"}, "product_id": 6644278001835, "shopify_product_id": 6644278001835, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "I Make Beats Wool Blend Snapback", "updated_at": "2021-05-13T07:27:34"}, "emitted_at": 1680895030371}
 {"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:20:10", "discount_amount": 0.0, "discount_type": "percentage", "handle": "new-mug", "id": 1853655, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_small.jpg"}, "product_id": 6688261701803, "shopify_product_id": 6688261701803, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "NEW!!! MUG", "updated_at": "2021-05-13T08:20:10"}, "emitted_at": 1680895030371}
 {"stream": "shop", "data": {"shop": {"allow_customers_to_skip_delivery": 1, "checkout_logo_url": null, "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Wed, 05 Apr 2023 02:44:22 GMT"}, "store": {"checkout_logo_url": null, "checkout_platform": "shopify", "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Wed, 05 Apr 2023 02:44:22 GMT"}}, "emitted_at": 1680895031312}

diff --git a/airbyte-integrations/connectors/source-recharge/metadata.yaml b/airbyte-integrations/connectors/source-recharge/metadata.yaml
@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 45d2e135-2ede-49e1-939f-3e3ec357a65e
-  dockerImageTag: 0.2.9
+  dockerImageTag: 0.2.10
   dockerRepository: airbyte/source-recharge
   githubIssueLabel: source-recharge
   icon: recharge.svg

diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/__init__.py b/airbyte-integrations/connectors/source-recharge/source_recharge/__init__.py
@@ -1,25 +1,5 @@
 #
-# MIT License
-#
-# Copyright (c) 2020 Airbyte
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
 

diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py
@@ -2,7 +2,6 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-
 from abc import ABC
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional
 
@@ -12,6 +11,9 @@
 from airbyte_cdk.sources.streams.http import HttpStream
 from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
 
+API_VERSION = "2021-11"
+OLD_API_VERSION = "2021-01"
+
 
 class RechargeStream(HttpStream, ABC):
     primary_key = "id"
@@ -22,7 +24,7 @@ class RechargeStream(HttpStream, ABC):
     period_in_months = 1  # Slice data request for 1 month
     raise_on_http_errors = True
 
-    # regestring the default schema transformation
+    # registering the default schema transformation
     transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization)
 
     def __init__(self, config, **kwargs):
@@ -33,28 +35,32 @@ def __init__(self, config, **kwargs):
     def data_path(self):
         return self.name
 
+    def request_headers(
+        self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
+    ) -> Mapping[str, Any]:
+        return {"x-recharge-version": API_VERSION}
+
     def path(
         self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
     ) -> str:
         return self.name
 
     def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
-        stream_data = self.get_stream_data(response.json())
-        if len(stream_data) == self.limit:
-            self.page_num += 1
-            return {"page": self.page_num}
+        cursor = response.json().get("next_cursor")
+        if cursor:
+            return {"cursor": cursor}
 
     def request_params(
         self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs
     ) -> MutableMapping[str, Any]:
         params = {
             "limit": self.limit,
-            "updated_at_min": (stream_slice or {}).get("start_date", self._start_date),
-            "updated_at_max": (stream_slice or {}).get("end_date", self._start_date),
         }
 
         if next_page_token:
             params.update(next_page_token)
+        else:
+            params.update({"updated_at_min": (stream_state or {}).get("updated_at", self._start_date)})
 
         return params
 
@@ -174,17 +180,29 @@ class Orders(IncrementalRechargeStream):
 class Products(RechargeStream):
     """
     Products Stream: https://developer.rechargepayments.com/v1-shopify?python#list-products
+    Products endpoint has 422 error with 2021-11 API version
     """
 
+    def request_headers(
+        self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
+    ) -> Mapping[str, Any]:
+        return {"x-recharge-version": OLD_API_VERSION}
+
 
 class Shop(RechargeStream):
     """
     Shop Stream: https://developer.rechargepayments.com/v1-shopify?python#shop
+    Shop endpoint is not available in 2021-11 API version
     """
 
     primary_key = ["shop", "store"]
     data_path = None
 
+    def request_headers(
+        self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
+    ) -> Mapping[str, Any]:
+        return {"x-recharge-version": OLD_API_VERSION}
+
 
 class Subscriptions(IncrementalRechargeStream):
     """