Skip to content

Commit

Permalink
馃悰 Source Shopify: Fix GraphQL query (#33827)
Browse files Browse the repository at this point in the history
  • Loading branch information
@artem1205
artem1205 committed Jan 8, 2024
1 parent c6e10a7 commit 6850437
Show file tree
Hide file tree
Showing 7 changed files with 44 additions and 23 deletions.
6 changes: 3 additions & 3 deletions airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ acceptance_tests:
basic_read:
tests:
- config_path: "secrets/config.json"
timeout_seconds: 3600
timeout_seconds: 4800
expect_records:
path: "integration_tests/expected_records.jsonl"
empty_streams:
Expand Down Expand Up @@ -68,12 +68,12 @@ acceptance_tests:
configured_catalog_path: "integration_tests/configured_catalog.json"
future_state:
future_state_path: "integration_tests/abnormal_state.json"
timeout_seconds: 3600
timeout_seconds: 14400
full_refresh:
tests:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
timeout_seconds: 3600
timeout_seconds: 4800
ignored_fields:
products:
- name: variants/*/updated_at
Expand Down
6 changes: 3 additions & 3 deletions airbyte-integrations/connectors/source-shopify/integration_tests/expected_records.jsonl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
{"stream": "inventory_levels", "data": {"inventory_item_id": 42185194668221, "location_id": 63590301885, "available": 12, "updated_at": "2021-06-22T18:09:27-07:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=42185194668221", "shop_url": "airbyte-integration-test", "id": "63590301885|42185194668221"}, "emitted_at": 1697194698578}
{"stream": "inventory_levels", "data": {"inventory_item_id": 42185194700989, "location_id": 63590301885, "available": 3, "updated_at": "2021-06-22T18:09:27-07:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=42185194700989", "shop_url": "airbyte-integration-test", "id": "63590301885|42185194700989"}, "emitted_at": 1697194698579}
{"stream": "inventory_levels", "data": {"inventory_item_id": 42185194733757, "location_id": 63590301885, "available": 38, "updated_at": "2021-06-22T18:09:27-07:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=42185194733757", "shop_url": "airbyte-integration-test", "id": "63590301885|42185194733757"}, "emitted_at": 1697194698579}
{"stream": "locations", "data": {"id": 63590301885, "name": "Heroiv UPA 72", "address1": "Heroiv UPA 72", "address2": null, "city": "Lviv", "zip": "30100", "province": null, "country": "UA", "phone": "", "created_at": "2021-06-22T18:00:29-07:00", "updated_at": "2023-02-25T16:20:00-08:00", "country_code": "UA", "country_name": "Ukraine", "province_code": null, "legacy": false, "active": true, "admin_graphql_api_id": "gid://shopify/Location/63590301885", "localized_country_name": "Ukraine", "localized_province_name": null, "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194701440}
{"stream":"locations","data":{"id":63590301885,"name":"Heroiv UPA 72","address1":"Heroiv UPA 72","address2":"","city":"Lviv","zip":"30100","province":null,"country":"UA","phone":"","created_at":"2021-06-22T18:00:29-07:00","updated_at":"2023-11-28T07:08:27-08:00","country_code":"UA","country_name":"Ukraine","province_code":null,"legacy":false,"active":true,"admin_graphql_api_id":"gid://shopify/Location/63590301885","localized_country_name":"Ukraine","localized_province_name":null,"shop_url":"airbyte-integration-test"},"emitted_at":1704314548257}
{"stream": "metafield_articles", "data": {"id": 21519818162365, "namespace": "global", "key": "new", "value": "newvalue", "description": null, "owner_id": 558137508029, "created_at": "2022-10-07T16:09:02-07:00", "updated_at": "2022-10-07T16:09:02-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21519818162365", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194703693}
{"stream": "metafield_articles", "data": {"id": 22365709992125, "namespace": "custom", "key": "test_blog_post_metafield", "value": "Test Article Metafield", "description": null, "owner_id": 558137508029, "created_at": "2023-04-14T03:18:26-07:00", "updated_at": "2023-04-14T03:18:26-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365709992125", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194703694}
{"stream": "metafield_articles", "data": {"id": 22365710352573, "namespace": "custom", "key": "test_blog_post_metafield", "value": "Test Blog Post Metafiled", "description": null, "owner_id": 558627979453, "created_at": "2023-04-14T03:19:18-07:00", "updated_at": "2023-04-14T03:19:18-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365710352573", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194704159}
Expand All @@ -36,8 +36,8 @@
{"stream": "metafield_customers", "data": {"id": 22346893361341, "namespace": "custom", "key": "test_definition_list_1", "value": "Teste\n", "description": null, "owner_id": 6569096478909, "created_at": "2023-04-13T04:50:10-07:00", "updated_at": "2023-04-13T04:50:10-07:00", "owner_resource": "customer", "type": "multi_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22346893361341", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194711312}
{"stream": "metafield_customers", "data": {"id": 22346893394109, "namespace": "custom", "key": "test_definition", "value": "Taster", "description": null, "owner_id": 6569096478909, "created_at": "2023-04-13T04:50:10-07:00", "updated_at": "2023-04-13T04:50:10-07:00", "owner_resource": "customer", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22346893394109", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194711313}
{"stream": "metafield_draft_orders", "data": {"id": 22532787175613, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 929019691197, "created_at": "2023-04-24T07:18:06-07:00", "updated_at": "2023-04-24T07:18:06-07:00", "owner_resource": "draft_order", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22532787175613", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194714876}
{"stream": "metafield_locations", "data": {"id": 21524407255229, "namespace": "inventory", "key": "warehouse_2", "value": "234", "description": null, "owner_id": 63590301885, "created_at": "2022-10-12T02:21:27-07:00", "updated_at": "2022-10-12T02:21:27-07:00", "owner_resource": "location", "type": "number_integer", "admin_graphql_api_id": "gid://shopify/Metafield/21524407255229", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194717772}
{"stream": "metafield_locations", "data": {"id": 21524407681213, "namespace": "inventory", "key": "warehouse_233", "value": "564", "description": null, "owner_id": 63590301885, "created_at": "2022-10-12T02:21:35-07:00", "updated_at": "2022-10-12T02:21:35-07:00", "owner_resource": "location", "type": "number_integer", "admin_graphql_api_id": "gid://shopify/Metafield/21524407681213", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194717773}
{"stream":"metafield_locations","data":{"id":21524407255229,"namespace":"inventory","key":"warehouse_2","value":"234","description":null,"owner_id":63590301885,"created_at":"2022-10-12T02:21:27-07:00","updated_at":"2022-10-12T02:21:27-07:00","owner_resource":"location","type":"number_integer","admin_graphql_api_id":"gid://shopify/Metafield/21524407255229","shop_url":"airbyte-integration-test"},"emitted_at":1704314554082}
{"stream":"metafield_locations","data":{"id":21524407681213,"namespace":"inventory","key":"warehouse_233","value":"564","description":null,"owner_id":63590301885,"created_at":"2022-10-12T02:21:35-07:00","updated_at":"2022-10-12T02:21:35-07:00","owner_resource":"location","type":"number_integer","admin_graphql_api_id":"gid://shopify/Metafield/21524407681213","shop_url":"airbyte-integration-test"},"emitted_at":1704314554084}
{"stream": "metafield_orders", "data": {"id": 22347287855293, "namespace": "my_fields", "key": "purchase_order", "value": "trtrtr", "description": null, "owner_id": 4147980107965, "created_at": "2023-04-13T05:09:08-07:00", "updated_at": "2023-04-13T05:09:08-07:00", "owner_resource": "order", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22347287855293", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194720313}
{"stream": "metafield_orders", "data": {"id": 22365749805245, "namespace": "my_fields", "key": "purchase_order", "value": "Test Draft Order Metafield", "description": null, "owner_id": 3935377129661, "created_at": "2023-04-14T03:52:40-07:00", "updated_at": "2023-04-14T03:52:40-07:00", "owner_resource": "order", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365749805245", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194720780}
{"stream": "metafield_pages", "data": {"id": 22534014828733, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 83074252989, "created_at": "2023-04-24T11:08:41-07:00", "updated_at": "2023-04-24T11:08:41-07:00", "owner_resource": "page", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22534014828733", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194723743}
Expand Down
4 changes: 2 additions & 2 deletions airbyte-integrations/connectors/source-shopify/metadata.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ data:
- ${shop}.myshopify.com
- shopify.com
connectorBuildOptions:
baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c
baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9
connectorSubtype: api
connectorType: source
definitionId: 9da77001-af33-4bcd-be46-6252bf9342b9
dockerImageTag: 1.1.4
dockerImageTag: 1.1.5
dockerRepository: airbyte/source-shopify
documentationUrl: https://docs.airbyte.com/integrations/sources/shopify
githubIssueLabel: source-shopify
Expand Down
10 changes: 6 additions & 4 deletions airbyte-integrations/connectors/source-shopify/source_shopify/graphql.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,12 @@ def _camel_to_snake(camel_case: str):
def get_query_products(first: int, filter_field: str, filter_value: str, next_page_token: Optional[str]):
op = sgqlc.operation.Operation(_schema_root.query_type)
snake_case_filter_field = _camel_to_snake(filter_field)
if next_page_token:
products = op.products(first=first, query=f"{snake_case_filter_field}:>'{filter_value}'", after=next_page_token)
else:
products = op.products(first=first, query=f"{snake_case_filter_field}:>'{filter_value}'")
products_args = {
"first": first,
"query": f"{snake_case_filter_field}:>'{filter_value}'" if filter_value else None,
"after": next_page_token,
}
products = op.products(**products_args)
products.nodes.id()
products.nodes.title()
products.nodes.updated_at()
Expand Down
2 changes: 2 additions & 0 deletions airbyte-integrations/connectors/source-shopify/source_shopify/source.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) ->
else:
params["order"] = f"{self.order_field} asc"
params[self.filter_field] = self.default_filter_field_value
if self.config.get("end_date") and self.filter_field == "updated_at_min":
params["updated_at_max"] = self.config.get("end_date")
return params

@limiter.balance_rate_limit()
Expand Down
16 changes: 16 additions & 0 deletions airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.

import pytest
from source_shopify.graphql import get_query_products


@pytest.mark.parametrize(
"page_size, filter_value, next_page_token, expected_query",
[
(100, None, None, 'query {\n products(first: 100, query: null, after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'),
(200, "2027-07-11T13:07:45-07:00", None, 'query {\n products(first: 200, query: "updated_at:>\'2027-07-11T13:07:45-07:00\'", after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'),
(250, "2027-07-11T13:07:45-07:00", "end_cursor_value", 'query {\n products(first: 250, query: "updated_at:>\'2027-07-11T13:07:45-07:00\'", after: "end_cursor_value") {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'),
],
)
def test_get_query_products(page_size, filter_value, next_page_token, expected_query):
assert get_query_products(page_size, 'updatedAt', filter_value, next_page_token) == expected_query
23 changes: 12 additions & 11 deletions docs/integrations/sources/shopify.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,17 +209,18 @@ If a child stream is synced independently of its parent stream, a full sync will
## Changelog

| Version | Date | Pull Request | Subject |
| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------ |
| 1.1.4 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image |
| 1.1.3 | 2023-10-17 | [31500](https://github.com/airbytehq/airbyte/pull/31500) | Fixed the issue caused by the `missing access token` while setup the new source and not yet authenticated |
| 1.1.2 | 2023-10-13 | [31381](https://github.com/airbytehq/airbyte/pull/31381) | Fixed the issue caused by the `state` presence while fetching the `deleted events` with pagination |
| 1.1.1 | 2023-09-18 | [30560](https://github.com/airbytehq/airbyte/pull/30560) | Performance testing - include socat binary in docker image |
| 1.1.0 | 2023-09-07 | [30246](https://github.com/airbytehq/airbyte/pull/30246) | Added ability to fetch `destroyed` records for `Articles, Blogs, CustomCollections, Orders, Pages, PriceRules, Products` |
| 1.0.0 | 2023-08-11 | [29361](https://github.com/airbytehq/airbyte/pull/29361) | Migrate to the `2023-07` Shopify API Version |
| 0.6.2 | 2023-08-09 | [29302](https://github.com/airbytehq/airbyte/pull/29302) | Handle the `Internal Server Error` when entity could be fetched |
| 0.6.1 | 2023-08-08 | [28291](https://github.com/airbytehq/airbyte/pull/28291) | Allow `shop` field to accept `*.myshopify.com` shop names, updated `OAuth Spec` |
| 0.6.0 | 2023-08-02 | [28770](https://github.com/airbytehq/airbyte/pull/28770) | Added `Disputes` stream |
| 0.5.1 | 2023-07-13 | [28700](https://github.com/airbytehq/airbyte/pull/28700) | Improved `error messages` with more user-friendly description, refactored code |
|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------|
| 1.1.5 | 2023-12-28 | [33827](https://github.com/airbytehq/airbyte/pull/33827) | Fix GraphQL query |
| 1.1.4 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image |
| 1.1.3 | 2023-10-17 | [31500](https://github.com/airbytehq/airbyte/pull/31500) | Fixed the issue caused by the `missing access token` while setup the new source and not yet authenticated |
| 1.1.2 | 2023-10-13 | [31381](https://github.com/airbytehq/airbyte/pull/31381) | Fixed the issue caused by the `state` presence while fetching the `deleted events` with pagination |
| 1.1.1 | 2023-09-18 | [30560](https://github.com/airbytehq/airbyte/pull/30560) | Performance testing - include socat binary in docker image |
| 1.1.0 | 2023-09-07 | [30246](https://github.com/airbytehq/airbyte/pull/30246) | Added ability to fetch `destroyed` records for `Articles, Blogs, CustomCollections, Orders, Pages, PriceRules, Products` |
| 1.0.0 | 2023-08-11 | [29361](https://github.com/airbytehq/airbyte/pull/29361) | Migrate to the `2023-07` Shopify API Version |
| 0.6.2 | 2023-08-09 | [29302](https://github.com/airbytehq/airbyte/pull/29302) | Handle the `Internal Server Error` when entity could be fetched |
| 0.6.1 | 2023-08-08 | [28291](https://github.com/airbytehq/airbyte/pull/28291) | Allow `shop` field to accept `*.myshopify.com` shop names, updated `OAuth Spec` |
| 0.6.0 | 2023-08-02 | [28770](https://github.com/airbytehq/airbyte/pull/28770) | Added `Disputes` stream |
| 0.5.1 | 2023-07-13 | [28700](https://github.com/airbytehq/airbyte/pull/28700) | Improved `error messages` with more user-friendly description, refactored code |
| 0.5.0 | 2023-06-13 | [27732](https://github.com/airbytehq/airbyte/pull/27732) | License Update: Elv2 |
| 0.4.0 | 2023-06-13 | [27083](https://github.com/airbytehq/airbyte/pull/27083) | Added `CustomerSavedSearch`, `CustomerAddress` and `Countries` streams |
| 0.3.4 | 2023-05-10 | [25961](https://github.com/airbytehq/airbyte/pull/25961) | Added validation for `shop` in input configuration (accepts non-url-like inputs) |
Expand Down

0 comments on commit 6850437

Please sign in to comment.