From c1e3a8ac09a0a8302a4a05338fa9f2bcf32dabfc Mon Sep 17 00:00:00 2001 From: Daryna Ishchenko Date: Tue, 20 Jun 2023 12:24:56 +0300 Subject: [PATCH 1/8] migrated to 2021-11 api version --- .../connectors/source-recharge/Dockerfile | 2 +- .../integration_tests/expected_records.jsonl | 6 +- .../connectors/source-recharge/metadata.yaml | 2 +- .../source-recharge/source_recharge/api.py | 34 ++++++++--- .../source_recharge/schemas/charges.json | 3 +- .../source-recharge/unit_tests/test_api.py | 60 ++++++++++--------- docs/integrations/sources/recharge.md | 41 ++++++------- 7 files changed, 83 insertions(+), 65 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/Dockerfile b/airbyte-integrations/connectors/source-recharge/Dockerfile index 0b8251340fe49..cea6ab99afd6f 100644 --- a/airbyte-integrations/connectors/source-recharge/Dockerfile +++ b/airbyte-integrations/connectors/source-recharge/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.9 +LABEL io.airbyte.version=0.2.10 LABEL io.airbyte.name=airbyte/source-recharge diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl index 321e32769096f..5c192d0d884c8 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl @@ -1,6 +1,6 @@ -{"stream": "addresses", "data": {"address1": "1 9th Ave", "address2": "1", "cart_attributes": null, "cart_note": null, "city": "San Francisco", "company": null, "country": "United States", "country_code": "US", "created_at": "2021-05-12T08:04:06", "customer_id": 64817252, "discount_id": null, "first_name": "Jane", "id": 69105381, "last_name": "Doe", "note_attributes": null, "original_shipping_lines": null, "phone": "1234567890", "presentment_currency": "USD", "province": "California", "shipping_lines_override": null, "updated_at": "2023-01-16T04:59:09", "zip": "94118"}, "emitted_at": 1680895024611} -{"stream": "charges", "data": {"address_id": 69105381, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country": "United States", "first_name": "Karina", "last_name": "Kuznetsova", "phone": null, "province": "California", "zip": "94118"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2021-05-12T08:04:07", "currency": "USD", "customer_hash": "23dee52d73734a81", "customer_id": 64817252, "discount_codes": [], "email": "nikolaevaka@yahoo.com", "error": "None\r\n [May 12, 12:06AM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 13, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 19, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 25, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 31, 4:09PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [Jun 06, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']", "error_type": "CLOSED_MAX_RETRIES_REACHED", "first_name": "Karina", "has_uncommited_changes": false, "id": 386976088, "last_charge_attempt_date": "2022-06-06T16:10:19", "last_name": "Kuznetsova", "line_items": [{"grams": 0, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "price": "24.30", "properties": [], "quantity": 1, "shopify_product_id": "6642695864491", "shopify_variant_id": "39684722131115", "sku": "T1", "subscription_id": 153224593, "tax_lines": [], "title": "Airbit Box Corner Short sleeve t-shirt", "type": "SUBSCRIPTION", "variant_title": "S / Black", "vendor": "Airbyte"}], "note": "", "note_attributes": null, "number_times_tried": 6, "processor_name": "shopify_payments", "requires_shipping": true, "retry_date": "2022-06-12T00:00:00", "scheduled_at": "2022-05-12T00:00:00", "shipments_count": null, "shipping_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country": "United States", "first_name": "Jane", "last_name": "Doe", "phone": "1234567890", "province": "California", "zip": "94118"}, "shipping_lines": [{"code": "Economy", "description": "", "price": 4.9, "source": "shopify", "tax_lines": [], "title": "Economy"}], "shopify_order_id": null, "shopify_variant_id_not_found": null, "status": "ERROR", "sub_total": null, "subtotal_price": "24.3", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "0.0", "total_discounts": "0.00", "total_duties": 0.0, "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": null, "total_tax": "0.0", "total_weight": 0, "transaction_id": null, "type": "RECURRING", "updated_at": "2023-01-16T13:08:54"}, "emitted_at": 1680895025756} -{"stream": "customers", "data": {"accepts_marketing": 1, "analytics_data": {"utm_params": []}, "billing_address1": "1 9th Ave", "billing_address2": "1", "billing_city": "San Francisco", "billing_company": null, "billing_country": "United States", "billing_phone": null, "billing_province": "California", "billing_zip": "94118", "created_at": "2021-05-12T08:04:06", "email": "nikolaevaka@yahoo.com", "first_charge_processed_at": "2021-05-12T12:03:59", "first_name": "Karina", "has_card_error_in_dunning": false, "has_valid_payment_method": true, "hash": "23dee52d73734a81", "id": 64817252, "last_name": "Kuznetsova", "number_active_subscriptions": 0, "number_subscriptions": 1, "phone": null, "processor_type": "shopify_payments", "reason_payment_method_not_valid": null, "shopify_customer_id": "5212085977259", "status": "ACTIVE", "tax_exempt": false, "updated_at": "2023-01-16T13:08:45"}, "emitted_at": 1680895026982} +{"stream": "addresses", "data": {"id": 69105381, "customer_id": 64817252, "payment_method_id": 12482012, "address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "created_at": "2021-05-12T12:04:06+00:00", "discounts": [], "first_name": "Jane", "last_name": "Doe", "order_attributes": [], "order_note": null, "phone": "1234567890", "presentment_currency": "USD", "province": "California", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-01-16T09:59:09+00:00", "zip": "94118"}, "emitted_at": 1680895024611} +{"stream": "charges", "data": {"id": 386976088, "address_id": 69105381, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Karina", "last_name": "Kuznetsova", "phone": null, "province": "California", "zip": "94118"}, "charge_attempts": 6, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2021-05-12T12:04:07+00:00", "currency": "USD", "customer": {"id": 64817252, "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "hash": "23dee52d73734a81"}, "discounts": [], "error": "None\r\n [May 12, 12:06AM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 13, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 19, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 25, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 31, 4:09PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [Jun 06, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']", "error_type": "CLOSED_MAX_RETRIES_REACHED", "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "external_variant_not_found": null, "has_uncommitted_changes": false, "last_charge_attempt": "2022-06-06T20:10:19+00:00", "line_items": [{"purchase_item_id": 153224593, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684722131115"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T1", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "S / Black"}], "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": "2022-06-12T04:00:00+00:00", "scheduled_at": "2022-05-12", "shipping_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Jane", "last_name": "Doe", "phone": "1234567890", "province": "California", "zip": "94118"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "source": "shopify", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "error", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-01-16T18:08:54+00:00"}, "emitted_at": 1687184458990} +{"stream": "customers", "data": {"id": 64817252, "analytics_data": {"utm_params": []}, "created_at": "2021-05-12T12:04:06+00:00", "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "first_charge_processed_at": "2021-05-12T16:03:59+00:00", "first_name": "Karina", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "23dee52d73734a81", "last_name": "Kuznetsova", "phone": null, "subscriptions_active_count": 0, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-01-16T18:08:45+00:00"}, "emitted_at": 1687184599794} {"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T07:27:34", "discount_amount": 5.0, "discount_type": "percentage", "handle": "i-make-beats-wool-blend-snapback", "id": 1853639, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_small.jpg"}, "product_id": 6644278001835, "shopify_product_id": 6644278001835, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "I Make Beats Wool Blend Snapback", "updated_at": "2021-05-13T07:27:34"}, "emitted_at": 1680895030371} {"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:20:10", "discount_amount": 0.0, "discount_type": "percentage", "handle": "new-mug", "id": 1853655, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_small.jpg"}, "product_id": 6688261701803, "shopify_product_id": 6688261701803, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "NEW!!! MUG", "updated_at": "2021-05-13T08:20:10"}, "emitted_at": 1680895030371} {"stream": "shop", "data": {"shop": {"allow_customers_to_skip_delivery": 1, "checkout_logo_url": null, "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Wed, 05 Apr 2023 02:44:22 GMT"}, "store": {"checkout_logo_url": null, "checkout_platform": "shopify", "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Wed, 05 Apr 2023 02:44:22 GMT"}}, "emitted_at": 1680895031312} diff --git a/airbyte-integrations/connectors/source-recharge/metadata.yaml b/airbyte-integrations/connectors/source-recharge/metadata.yaml index 5bd49ed954116..1ff1bb6e0d324 100644 --- a/airbyte-integrations/connectors/source-recharge/metadata.yaml +++ b/airbyte-integrations/connectors/source-recharge/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 45d2e135-2ede-49e1-939f-3e3ec357a65e - dockerImageTag: 0.2.9 + dockerImageTag: 0.2.10 dockerRepository: airbyte/source-recharge githubIssueLabel: source-recharge icon: recharge.svg diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py index 204975e60602d..bde597bc1a6af 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py @@ -12,6 +12,9 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +API_VERSION = "2021-11" +OLD_API_VERSION = "2021-01" + class RechargeStream(HttpStream, ABC): primary_key = "id" @@ -22,7 +25,7 @@ class RechargeStream(HttpStream, ABC): period_in_months = 1 # Slice data request for 1 month raise_on_http_errors = True - # regestring the default schema transformation + # registering the default schema transformation transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) def __init__(self, config, **kwargs): @@ -33,16 +36,20 @@ def __init__(self, config, **kwargs): def data_path(self): return self.name + def request_headers( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + return {"x-recharge-version": API_VERSION} + def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: return self.name def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - stream_data = self.get_stream_data(response.json()) - if len(stream_data) == self.limit: - self.page_num += 1 - return {"page": self.page_num} + cursor = response.json().get("next_cursor") + if cursor: + return {"cursor": cursor} def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs @@ -50,7 +57,6 @@ def request_params( params = { "limit": self.limit, "updated_at_min": (stream_slice or {}).get("start_date", self._start_date), - "updated_at_max": (stream_slice or {}).get("end_date", self._start_date), } if next_page_token: @@ -89,9 +95,7 @@ def stream_slices( start_date = pendulum.parse(start_date) while start_date <= now: - end_date = start_date.add(months=self.period_in_months) - yield {"start_date": start_date.strftime("%Y-%m-%d %H:%M:%S"), "end_date": end_date.strftime("%Y-%m-%d %H:%M:%S")} - start_date = end_date + yield {"start_date": start_date.strftime("%Y-%m-%d %H:%M:%S")} class IncrementalRechargeStream(RechargeStream, ABC): @@ -174,17 +178,29 @@ class Orders(IncrementalRechargeStream): class Products(RechargeStream): """ Products Stream: https://developer.rechargepayments.com/v1-shopify?python#list-products + Products endpoint has 422 error with 2021-11 API version """ + def request_headers( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + return {"x-recharge-version": OLD_API_VERSION} + class Shop(RechargeStream): """ Shop Stream: https://developer.rechargepayments.com/v1-shopify?python#shop + Shop endpoint is not available in 2021-11 API version """ primary_key = ["shop", "store"] data_path = None + def request_headers( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + return {"x-recharge-version": OLD_API_VERSION} + class Subscriptions(IncrementalRechargeStream): """ diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json index c3aa406e5cb88..29814d448a645 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json @@ -137,8 +137,7 @@ "format": "date-time" }, "scheduled_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "string"] }, "shipments_count": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py index dd552c6f656c8..ba4324a01c7df 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py @@ -191,29 +191,30 @@ def generate_records(self, stream_name, count): return {stream_name: result} @pytest.mark.parametrize( - "stream_cls, rec_limit, expected", + "stream_cls, cursor_response, expected", [ - (Collections, 1, {"page": 2}), - (Metafields, 2, {"page": 2}), - (Products, 1, {"page": 2}), - (Shop, 1, {"page": 2}), + (Collections, "some next cursor", {"cursor": "some next cursor"}), + (Metafields, "some next cursor", {"cursor": "some next cursor"}), + (Products, "some next cursor", {"cursor": "some next cursor"}), + (Shop, "some next cursor", {"cursor": "some next cursor"}), ], ) - def test_next_page_token(self, config, stream_cls, rec_limit, requests_mock, expected): + def test_next_page_token(self, config, stream_cls, cursor_response, requests_mock, expected): stream = stream_cls(config, authenticator=None) - stream.limit = rec_limit + stream.limit = 2 url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=self.generate_records(stream.name, rec_limit)) + response = {"next_cursor": cursor_response, stream.name: self.generate_records(stream.name, 2)} + requests_mock.get(url, json=response) response = requests.get(url) assert stream.next_page_token(response) == expected @pytest.mark.parametrize( "stream_cls, next_page_token, stream_state, stream_slice, expected", [ - (Collections, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), + (Collections, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), (Metafields, {"page": 2}, {"updated_at": "2030-01-01"}, {}, {"limit": 250, "owner_resource": None, "page": 2}), - (Products, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), - (Shop, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), + (Products, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), + (Shop, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z",}), ], ) def test_request_params(self, config, stream_cls, next_page_token, stream_state, stream_slice, expected): @@ -287,38 +288,39 @@ def test_cursor_field(self, config, stream_cls, expected): assert result == expected @pytest.mark.parametrize( - "stream_cls, rec_limit, expected", + "stream_cls, cursor_response, expected", [ - (Addresses, 1, {"page": 2}), - (Charges, 2, {"page": 2}), - (Customers, 1, {"page": 2}), - (Discounts, 1, {"page": 2}), - (Onetimes, 1, {"page": 2}), - (Orders, 1, {"page": 2}), - (Subscriptions, 1, {"page": 2}), + (Addresses, "some next cursor", {"cursor": "some next cursor"}), + (Charges, "some next cursor", {"cursor": "some next cursor"}), + (Customers, "some next cursor", {"cursor": "some next cursor"}), + (Discounts, "some next cursor", {"cursor": "some next cursor"}), + (Onetimes, "some next cursor", {"cursor": "some next cursor"}), + (Orders, "some next cursor", {"cursor": "some next cursor"}), + (Subscriptions, "some next cursor", {"cursor": "some next cursor"}), ], ) - def test_next_page_token(self, config, stream_cls, rec_limit, requests_mock, expected): + def test_next_page_token(self, config, stream_cls, cursor_response, requests_mock, expected): stream = stream_cls(config, authenticator=None) - stream.limit = rec_limit + stream.limit = 2 url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=self.generate_records(stream.name, rec_limit)) + response = {"next_cursor": cursor_response, stream.name: self.generate_records(stream.name, 2)} + requests_mock.get(url, json=response) response = requests.get(url) assert stream.next_page_token(response) == expected @pytest.mark.parametrize( "stream_cls, next_page_token, stream_state, stream_slice, expected", [ - (Addresses, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), + (Addresses, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), (Charges, {"page": 2}, {"updated_at": "2030-01-01"}, {}, - {"limit": 250, "page": 2, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), - (Customers, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), - (Discounts, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), + {"limit": 250, "page": 2, "updated_at_min": "2021-08-15T00:00:00Z"}), + (Customers, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), + (Discounts, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), (Onetimes, {"page": 2}, {"updated_at": "2030-01-01"}, {}, - {"limit": 250, "page": 2, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), - (Orders, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), + {"limit": 250, "page": 2, "updated_at_min": "2021-08-15T00:00:00Z"}), + (Orders, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), (Subscriptions, None, {}, {}, - {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), + {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), ], ) def test_request_params(self, config, stream_cls, next_page_token, stream_state, stream_slice, expected): diff --git a/docs/integrations/sources/recharge.md b/docs/integrations/sources/recharge.md index 19909f5ddcc4b..96c98bc7d80ba 100644 --- a/docs/integrations/sources/recharge.md +++ b/docs/integrations/sources/recharge.md @@ -74,23 +74,24 @@ The Recharge connector should gracefully handle Recharge API limitations under n ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------| -| 0.2.9 | 2023-04-10 | [25009](https://github.com/airbytehq/airbyte/pull/25009) | Fix owner slicing for `Metafields` stream | -| 0.2.8 | 2023-04-07 | [24990](https://github.com/airbytehq/airbyte/pull/24990) | Add slicing to connector | -| 0.2.7 | 2023-02-13 | [22901](https://github.com/airbytehq/airbyte/pull/22901) | Specified date formatting in specification | -| 0.2.6 | 2023-02-21 | [22473](https://github.com/airbytehq/airbyte/pull/22473) | Use default availability strategy | -| 0.2.5 | 2023-01-27 | [22021](https://github.com/airbytehq/airbyte/pull/22021) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.2.4 | 2022-10-11 | [17822](https://github.com/airbytehq/airbyte/pull/17822) | Do not parse JSON in `should_retry` | -| 0.2.3 | 2022-10-11 | [17822](https://github.com/airbytehq/airbyte/pull/17822) | Do not parse JSON in `should_retry` | -| 0.2.2 | 2022-10-05 | [17608](https://github.com/airbytehq/airbyte/pull/17608) | Skip stream if we receive 403 error | -| 0.2.2 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | -| 0.2.1 | 2022-09-23 | [17080](https://github.com/airbytehq/airbyte/pull/17080) | Fix `total_weight` value to be `int` instead of `float` | -| 0.2.0 | 2022-09-21 | [16959](https://github.com/airbytehq/airbyte/pull/16959) | Use TypeTransformer to reliably convert to schema declared data types | -| 0.1.8 | 2022-08-27 | [16045](https://github.com/airbytehq/airbyte/pull/16045) | Force total_weight to be an integer | -| 0.1.7 | 2022-07-24 | [14978](https://github.com/airbytehq/airbyte/pull/14978) | Set `additionalProperties` to True, to guarantee backward cababilities | -| 0.1.6 | 2022-07-21 | [14902](https://github.com/airbytehq/airbyte/pull/14902) | Increased test coverage, fixed broken `charges`, `orders` schemas, added state checkpoint | -| 0.1.5 | 2022-01-26 | [9808](https://github.com/airbytehq/airbyte/pull/9808) | Update connector fields title/description | -| 0.1.4 | 2021-11-05 | [7626](https://github.com/airbytehq/airbyte/pull/7626) | Improve 'backoff' for HTTP requests | -| 0.1.3 | 2021-09-17 | [6149](https://github.com/airbytehq/airbyte/pull/6149) | Update `discount` and `order` schema | -| 0.1.2 | 2021-09-17 | [6149](https://github.com/airbytehq/airbyte/pull/6149) | Change `cursor_field` for Incremental streams | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------| +| 0.2.10 | 2023-06-20 | | Update API version to 2021-11 | +| 0.2.9 | 2023-04-10 | [25009](https://github.com/airbytehq/airbyte/pull/25009) | Fix owner slicing for `Metafields` stream | +| 0.2.8 | 2023-04-07 | [24990](https://github.com/airbytehq/airbyte/pull/24990) | Add slicing to connector | +| 0.2.7 | 2023-02-13 | [22901](https://github.com/airbytehq/airbyte/pull/22901) | Specified date formatting in specification | +| 0.2.6 | 2023-02-21 | [22473](https://github.com/airbytehq/airbyte/pull/22473) | Use default availability strategy | +| 0.2.5 | 2023-01-27 | [22021](https://github.com/airbytehq/airbyte/pull/22021) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.2.4 | 2022-10-11 | [17822](https://github.com/airbytehq/airbyte/pull/17822) | Do not parse JSON in `should_retry` | +| 0.2.3 | 2022-10-11 | [17822](https://github.com/airbytehq/airbyte/pull/17822) | Do not parse JSON in `should_retry` | +| 0.2.2 | 2022-10-05 | [17608](https://github.com/airbytehq/airbyte/pull/17608) | Skip stream if we receive 403 error | +| 0.2.2 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | +| 0.2.1 | 2022-09-23 | [17080](https://github.com/airbytehq/airbyte/pull/17080) | Fix `total_weight` value to be `int` instead of `float` | +| 0.2.0 | 2022-09-21 | [16959](https://github.com/airbytehq/airbyte/pull/16959) | Use TypeTransformer to reliably convert to schema declared data types | +| 0.1.8 | 2022-08-27 | [16045](https://github.com/airbytehq/airbyte/pull/16045) | Force total_weight to be an integer | +| 0.1.7 | 2022-07-24 | [14978](https://github.com/airbytehq/airbyte/pull/14978) | Set `additionalProperties` to True, to guarantee backward cababilities | +| 0.1.6 | 2022-07-21 | [14902](https://github.com/airbytehq/airbyte/pull/14902) | Increased test coverage, fixed broken `charges`, `orders` schemas, added state checkpoint | +| 0.1.5 | 2022-01-26 | [9808](https://github.com/airbytehq/airbyte/pull/9808) | Update connector fields title/description | +| 0.1.4 | 2021-11-05 | [7626](https://github.com/airbytehq/airbyte/pull/7626) | Improve 'backoff' for HTTP requests | +| 0.1.3 | 2021-09-17 | [6149](https://github.com/airbytehq/airbyte/pull/6149) | Update `discount` and `order` schema | +| 0.1.2 | 2021-09-17 | [6149](https://github.com/airbytehq/airbyte/pull/6149) | Change `cursor_field` for Incremental streams | From 89e502efb3e87c947a42b3961014f3c325fbf75b Mon Sep 17 00:00:00 2001 From: octavia-squidington-iii Date: Tue, 20 Jun 2023 14:08:59 +0000 Subject: [PATCH 2/8] =?UTF-8?q?=F0=9F=A4=96=20Auto=20format=20source-recha?= =?UTF-8?q?rge=20code=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../integration_tests/__init__.py | 3 +++ .../source_recharge/__init__.py | 22 +------------------ 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/__init__.py b/airbyte-integrations/connectors/source-recharge/integration_tests/__init__.py index e69de29bb2d1d..c941b30457953 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/__init__.py b/airbyte-integrations/connectors/source-recharge/source_recharge/__init__.py index a7aafe9ffd60a..d5ab4df2ea0fb 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/__init__.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/__init__.py @@ -1,25 +1,5 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # From 03550d01fe422617523c236dcd5f89a74ac06b52 Mon Sep 17 00:00:00 2001 From: Daryna Ishchenko Date: Tue, 20 Jun 2023 17:12:44 +0300 Subject: [PATCH 3/8] updated changelog --- docs/integrations/sources/recharge.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/recharge.md b/docs/integrations/sources/recharge.md index 96c98bc7d80ba..bc0afa5fc3da1 100644 --- a/docs/integrations/sources/recharge.md +++ b/docs/integrations/sources/recharge.md @@ -76,7 +76,7 @@ The Recharge connector should gracefully handle Recharge API limitations under n | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------| -| 0.2.10 | 2023-06-20 | | Update API version to 2021-11 | +| 0.2.10 | 2023-06-20 | [27503](https://github.com/airbytehq/airbyte/pull/27503) | Update API version to 2021-11 | | 0.2.9 | 2023-04-10 | [25009](https://github.com/airbytehq/airbyte/pull/25009) | Fix owner slicing for `Metafields` stream | | 0.2.8 | 2023-04-07 | [24990](https://github.com/airbytehq/airbyte/pull/24990) | Add slicing to connector | | 0.2.7 | 2023-02-13 | [22901](https://github.com/airbytehq/airbyte/pull/22901) | Specified date formatting in specification | From 2bdb2cec5a840bfe3fab162433d218fc1248dcf6 Mon Sep 17 00:00:00 2001 From: Daryna Ishchenko Date: Wed, 21 Jun 2023 11:50:45 +0300 Subject: [PATCH 4/8] updated request params --- .../source-recharge/acceptance-test-config.yml | 2 +- .../connectors/source-recharge/source_recharge/api.py | 3 ++- .../connectors/source-recharge/unit_tests/test_api.py | 10 +++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml index 596b618eae095..21969eabdf028 100644 --- a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml @@ -19,7 +19,7 @@ acceptance_tests: bypass_reason: "updated after login" - name: store/updated_at bypass_reason: "updated after login" - timeout_seconds: 1200 + timeout_seconds: 3600 expect_records: path: "integration_tests/expected_records.jsonl" extra_fields: no diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py index bde597bc1a6af..3e20cc20b6558 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py @@ -56,11 +56,12 @@ def request_params( ) -> MutableMapping[str, Any]: params = { "limit": self.limit, - "updated_at_min": (stream_slice or {}).get("start_date", self._start_date), } if next_page_token: params.update(next_page_token) + else: + params.update({"updated_at_min": (stream_slice or {}).get("start_date", self._start_date)}) return params diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py index ba4324a01c7df..6e1cc27e492ac 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py @@ -212,7 +212,7 @@ def test_next_page_token(self, config, stream_cls, cursor_response, requests_moc "stream_cls, next_page_token, stream_state, stream_slice, expected", [ (Collections, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), - (Metafields, {"page": 2}, {"updated_at": "2030-01-01"}, {}, {"limit": 250, "owner_resource": None, "page": 2}), + (Metafields, {"cursor": "12353"}, {"updated_at": "2030-01-01"}, {}, {"limit": 250, "owner_resource": None, "cursor": "12353"}), (Products, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), (Shop, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z",}), ], @@ -312,12 +312,12 @@ def test_next_page_token(self, config, stream_cls, cursor_response, requests_moc "stream_cls, next_page_token, stream_state, stream_slice, expected", [ (Addresses, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), - (Charges, {"page": 2}, {"updated_at": "2030-01-01"}, {}, - {"limit": 250, "page": 2, "updated_at_min": "2021-08-15T00:00:00Z"}), + (Charges, {"cursor": "123"}, {"updated_at": "2030-01-01"}, {}, + {"limit": 250, "cursor": "123"}), (Customers, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), (Discounts, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), - (Onetimes, {"page": 2}, {"updated_at": "2030-01-01"}, {}, - {"limit": 250, "page": 2, "updated_at_min": "2021-08-15T00:00:00Z"}), + (Onetimes, {"cursor": "123"}, {"updated_at": "2030-01-01"}, {}, + {"limit": 250, "cursor": "123"}), (Orders, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), (Subscriptions, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), From 6d44a9bc5b748720cc1addef00cc23f9e7181116 Mon Sep 17 00:00:00 2001 From: Daryna Ishchenko Date: Wed, 21 Jun 2023 13:56:15 +0300 Subject: [PATCH 5/8] increased timeout for sat --- .../connectors/source-recharge/acceptance-test-config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml index 21969eabdf028..ecf75a9852149 100644 --- a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml @@ -19,7 +19,7 @@ acceptance_tests: bypass_reason: "updated after login" - name: store/updated_at bypass_reason: "updated after login" - timeout_seconds: 3600 + timeout_seconds: 7200 expect_records: path: "integration_tests/expected_records.jsonl" extra_fields: no @@ -41,14 +41,14 @@ acceptance_tests: tests: - config_path: secrets/config.json configured_catalog_path: integration_tests/configured_catalog.json - timeout_seconds: 1200 + timeout_seconds: 2400 incremental: tests: - config_path: secrets/config.json configured_catalog_path: integration_tests/streams_with_output_records_catalog.json future_state: future_state_path: integration_tests/abnormal_state.json - timeout_seconds: 900 + timeout_seconds: 2400 spec: tests: - spec_path: source_recharge/spec.json From 42bd9d98702acfdceef5d8913b60f41424f092df Mon Sep 17 00:00:00 2001 From: Daryna Ishchenko Date: Wed, 21 Jun 2023 18:19:07 +0300 Subject: [PATCH 6/8] increased timeout for sat --- .../connectors/source-recharge/acceptance-test-config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml index ecf75a9852149..9e8b757d0d14d 100644 --- a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml @@ -41,14 +41,14 @@ acceptance_tests: tests: - config_path: secrets/config.json configured_catalog_path: integration_tests/configured_catalog.json - timeout_seconds: 2400 + timeout_seconds: 3200 incremental: tests: - config_path: secrets/config.json configured_catalog_path: integration_tests/streams_with_output_records_catalog.json future_state: future_state_path: integration_tests/abnormal_state.json - timeout_seconds: 2400 + timeout_seconds: 3200 spec: tests: - spec_path: source_recharge/spec.json From cc6ef090806cae1d24aebcf9122213bf83a1966a Mon Sep 17 00:00:00 2001 From: Daryna Ishchenko Date: Fri, 23 Jun 2023 19:02:01 +0300 Subject: [PATCH 7/8] removed stream slices --- .../source-recharge/source_recharge/api.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py index 3e20cc20b6558..1befbbc36b482 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py @@ -2,11 +2,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional -import pendulum import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream @@ -61,7 +59,7 @@ def request_params( if next_page_token: params.update(next_page_token) else: - params.update({"updated_at_min": (stream_slice or {}).get("start_date", self._start_date)}) + params.update({"updated_at_min": (stream_state or {}).get("updated_at", self._start_date)}) return params @@ -86,18 +84,6 @@ def should_retry(self, response: requests.Response) -> bool: return super().should_retry(response) - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - start_date = (stream_state or {}).get(self.cursor_field, self._start_date) if self.cursor_field else self._start_date - - now = pendulum.now() - - start_date = pendulum.parse(start_date) - - while start_date <= now: - yield {"start_date": start_date.strftime("%Y-%m-%d %H:%M:%S")} - class IncrementalRechargeStream(RechargeStream, ABC): cursor_field = "updated_at" From 89e3baa968841a8d5700822a9b64f0676235426f Mon Sep 17 00:00:00 2001 From: Daryna Ishchenko Date: Mon, 26 Jun 2023 13:56:48 +0300 Subject: [PATCH 8/8] added stream_slices --- .../source-recharge/source_recharge/api.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py index 1befbbc36b482..c2c9d3e85f8c2 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py @@ -5,6 +5,7 @@ from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +import pendulum import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream @@ -84,6 +85,20 @@ def should_retry(self, response: requests.Response) -> bool: return super().should_retry(response) + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + start_date = (stream_state or {}).get(self.cursor_field, self._start_date) if self.cursor_field else self._start_date + + now = pendulum.now() + + start_date = pendulum.parse(start_date) + + while start_date <= now: + end_date = start_date.add(months=self.period_in_months) + yield {"start_date": start_date.strftime("%Y-%m-%d %H:%M:%S"), "end_date": end_date.strftime("%Y-%m-%d %H:%M:%S")} + start_date = end_date + class IncrementalRechargeStream(RechargeStream, ABC): cursor_field = "updated_at"