🐛 Source Shopify: Migrate from REST > GraphQL BULK Operations whe…

…re possible, fixed `STATE` collisions for `sub-streams` (#32345)
airbytehq · Feb 26, 2024 · f509404 · f509404
1 parent 5fc6ca2
commit f509404
Show file tree

Hide file tree

Showing 45 changed files with 13,306 additions and 4,405 deletions.
diff --git a/airbyte-integrations/connectors/source-shopify/README.md b/airbyte-integrations/connectors/source-shopify/README.md
@@ -30,7 +30,7 @@ See `sample_files/sample_config.json` for a sample config file.
 poetry run source-shopify spec
 poetry run source-shopify check --config secrets/config.json
 poetry run source-shopify discover --config secrets/config.json
-poetry run source-shopify read --config secrets/config.json --catalog sample_files/configured_catalog.json
+poetry run source-shopify read --config secrets/config.json --catalog integration_tests/configured_catalog.json
 ```
 
 ### Running unit tests

diff --git a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml
@@ -24,8 +24,47 @@ acceptance_tests:
   discovery:
     tests:
       - config_path: "secrets/config.json"
+        backward_compatibility_tests_config:
+          # The cursor field for `fulfillments` stream has changed from `id` to `updated_at`
+          disable_for_version: "1.1.8"
   basic_read:
     tests:
+      - config_path: "secrets/config_transactions_with_user_id.json"
+        timeout_seconds: 4800
+        expect_records:
+          path: "integration_tests/expected_records_transactions_with_user_id.jsonl"
+        empty_streams:
+          - name: abandoned_checkouts
+            bypass_reason: The stream holds data up to 1 month then records are removed by Shopify.
+          - name: balance_transactions
+            bypass_reason: The stream requires real purchases to fill in the data.
+          - name: customer_saved_search
+            bypass_reason: The stream is not available for our sandbox.
+          - name: disputes
+            bypass_reason: The stream requires real purchases to fill in the data.
+        ignored_fields:
+          products:
+            - name: variants/*/updated_at
+              bypass_reason: Value can change as the account data is not frozen
+            - name: image/src
+              bypass_reason: May contain dynamically changed URL params
+            - name: image/updated_at
+              bypass_reason: Value can change as the account data is not frozen
+            - name: images/*/src
+              bypass_reason: May contain dynamically changed URL params
+            - name: images/*/updated_at
+              bypass_reason: Value can change as the account data is not frozen
+          products_graph_ql:
+            - name: onlineStorePreviewUrl
+              bypass_reason: Autogenerated floating URL values
+          product_variants:
+            - name: updated_at
+              bypass_reason: Value can change as the account data is not frozen
+          product_images:
+            - name: src
+              bypass_reason: May contain dynamically changed URL params
+            - name: updated_at
+              bypass_reason: Value can change as the account data is not frozen
       - config_path: "secrets/config.json"
         timeout_seconds: 4800
         expect_records:
@@ -68,7 +107,7 @@ acceptance_tests:
         configured_catalog_path: "integration_tests/configured_catalog.json"
         future_state:
           future_state_path: "integration_tests/abnormal_state.json"
-        timeout_seconds: 14400
+        timeout_seconds: 7200
   full_refresh:
     tests:
       - config_path: "secrets/config.json"

diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json
@@ -158,6 +158,17 @@
       }
     }
   },
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "created_at": "2027-07-11T13:07:45-07:00"
+      },
+      "stream_descriptor": {
+        "name": "transactions_graphql"
+      }
+    }
+  },
   {
     "type": "STREAM",
     "stream": {
@@ -253,7 +264,7 @@
             "deleted_at": "2027-07-11T13:07:45-07:00"
           }
         },
-        "id": 99999999999999
+        "updated_at": "2027-07-11T13:07:45-07:00"
       },
       "stream_descriptor": {
         "name": "fulfillment_orders"
@@ -449,7 +460,8 @@
           "deleted": {
             "deleted_at": "2027-07-11T13:07:45-07:00"
           }
-        }
+        },
+        "updated_at": "2027-07-11T13:07:45-07:00"
       },
       "stream_descriptor": {
         "name": "product_variants"

diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-shopify/integration_tests/expected_records.jsonl
diff --git a/...nectors/source-shopify/integration_tests/expected_records_transactions_with_user_id.jsonl b/...nectors/source-shopify/integration_tests/expected_records_transactions_with_user_id.jsonl
diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json
@@ -174,13 +174,13 @@
     "updated_at": "2023-04-14T04:04:46-07:00"
   },
   "product_images": {
-    "id": 33290489659581,
     "products": {
-      "updated_at": "",
+      "updated_at": "2023-04-24T11:05:13-07:00",
       "deleted": {
         "deleted_at": "2023-09-05T13:32:22-07:00"
       }
-    }
+    },
+    "updated_at": "2023-04-24T10:27:15-07:00"
   },
   "metafield_product_images": {
     "products": {

diff --git a/airbyte-integrations/connectors/source-shopify/metadata.yaml b/airbyte-integrations/connectors/source-shopify/metadata.yaml
@@ -11,7 +11,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 9da77001-af33-4bcd-be46-6252bf9342b9
-  dockerImageTag: 1.1.8
+  dockerImageTag: 2.0.0
   dockerRepository: airbyte/source-shopify
   documentationUrl: https://docs.airbyte.com/integrations/sources/shopify
   githubIssueLabel: source-shopify
@@ -35,11 +35,37 @@ data:
           "This upgrade brings changes to certain streams after migration to
           Shopify API version `2023-07`, more details in this PR: https://github.com/airbytehq/airbyte/pull/29361."
         upgradeDeadline: "2023-09-17"
+      2.0.0:
+        message: "This upgrade brings perfomance impovements and stream schema changes. Details are available here: https://github.com/airbytehq/airbyte/pull/32345#issue-1985556333."
+        upgradeDeadline: "2024-03-18"
+        scopedImpact:
+          - scopeType: stream
+            impactedScopes:
+              [
+                "collections",
+                "customer_address",
+                "discount_codes",
+                "fulfillment_orders",
+                "fulfillments",
+                "inventory_items",
+                "inventory_levels",
+                "metafield_collections",
+                "metafield_customers",
+                "metafield_draft_orders",
+                "metafield_locations",
+                "metafield_orders",
+                "metafield_product_images",
+                "metafield_product_variants",
+                "order_refunds",
+                "product_images",
+                "product_variants",
+                "transactions",
+              ]
   suggestedStreams:
     streams:
       - customers
       - order_refunds
-      - transactions
+      - transactions_graphql
       - product_variants
       - abandoned_checkouts
       - discount_codes
@@ -50,7 +76,6 @@ data:
       - inventory_levels
       - custom_collections
       - fulfillments
-      - order_risks
       - products_graph_ql
       - product_images
       - collects

diff --git a/airbyte-integrations/connectors/source-shopify/poetry.lock b/airbyte-integrations/connectors/source-shopify/poetry.lock
diff --git a/airbyte-integrations/connectors/source-shopify/pyproject.toml b/airbyte-integrations/connectors/source-shopify/pyproject.toml
@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "1.1.8"
+version = "2.0.0"
 name = "source-shopify"
 description = "Source CDK implementation for Shopify."
 authors = [ "Airbyte <contact@airbyte.io>",]
@@ -19,6 +19,7 @@ include = "source_shopify"
 python = "^3.9,<3.12"
 airbyte-cdk = "==0.59.0"
 sgqlc = "==16.3"
+graphql-query = "^1.1.1"
 
 [tool.poetry.scripts]
 source-shopify = "source_shopify.run:run"

diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py b/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py
@@ -34,7 +34,6 @@ def __init__(self, config: Mapping[str, Any]):
         self.config = config
 
     def get_auth_header(self) -> Mapping[str, Any]:
-
         auth_header: str = "X-Shopify-Access-Token"
         credentials: Dict = self.config.get("credentials", self.config.get("auth_method"))
         auth_method: str = credentials.get("auth_method")

diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customer_address.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customer_address.json
@@ -55,6 +55,10 @@
     },
     "shop_url": {
       "type": ["null", "string"]
+    },
+    "updated_at": {
+      "type": ["null", "string"],
+      "format": "date-time"
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json
@@ -214,6 +214,9 @@
     "sms_marketing_consent": {
       "type": ["null", "object"],
       "properties": {
+        "consent_collected_from": {
+          "type": ["null", "string"]
+        },
         "consent_updated_at": {
           "type": ["null", "string"],
           "format": "date-time"

diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/discount_codes.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/discount_codes.json
@@ -22,6 +22,15 @@
       "type": ["null", "string"],
       "format": "date-time"
     },
+    "summary": {
+      "type": ["null", "string"]
+    },
+    "discount_type": {
+      "type": ["null", "string"]
+    },
+    "admin_graphql_api_id": {
+      "type": ["null", "string"]
+    },
     "shop_url": {
       "type": ["null", "string"]
     }

diff --git a/...yte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillment_orders.json b/...yte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillment_orders.json
@@ -54,6 +54,14 @@
         },
         "method_type": {
           "type": ["null", "string"]
+        },
+        "min_delivery_date_time": {
+          "type": ["null", "string"],
+          "format": "date-time"
+        },
+        "max_delivery_date_time": {
+          "type": ["null", "string"],
+          "format": "date-time"
         }
       }
     },
@@ -99,7 +107,7 @@
           "shop_id": {
             "type": ["null", "integer"]
           },
-          "fullfillment_order_id": {
+          "fulfillment_order_id": {
             "type": ["null", "integer"]
           },
           "line_item_id": {
@@ -135,22 +143,39 @@
     "supported_actions": {
       "type": ["null", "array"],
       "items": {
-        "type": ["null", "string"]
+        "type": ["null", "object"],
+        "properties": {
+          "action": {
+            "type": ["null", "string"]
+          },
+          "external_url": {
+            "type": ["null", "string"]
+          }
+        }
       }
     },
     "merchant_requests": {
       "type": ["null", "array"],
       "items": {
         "type": ["null", "object"],
         "properties": {
+          "id": {
+            "type": ["null", "integer"]
+          },
           "message": {
             "type": ["null", "string"]
           },
           "kind": {
             "type": ["null", "string"]
           },
           "request_options": {
-            "type": ["null", "object"]
+            "type": ["null", "object"],
+            "additionalProperties": true,
+            "properties": {
+              "notify_customer": {
+                "type": ["null", "boolean"]
+              }
+            }
           }
         }
       }
@@ -197,6 +222,9 @@
     "updated_at": {
       "type": ["null", "string"],
       "format": "date-time"
+    },
+    "admin_graphql_api_id": {
+      "type": ["null", "string"]
     }
   }
 }