🎉 Source Shopify - change the default stream cursor_field to 'updated…

…_at' where possible (#4472) #4472 - Source Shopify - change the default stream cursor_field to 'updated_at' where possible Co-authored-by: Oleksandr Bazarnov <oleksandr.bazarnov@globallogic.com>
airbytehq · Jul 5, 2021 · 9517fae · 9517fae
1 parent d382d1b
commit 9517fae
Show file tree

Hide file tree

Showing 10 changed files with 264 additions and 152 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json
@@ -2,6 +2,6 @@
   "sourceDefinitionId": "9da77001-af33-4bcd-be46-6252bf9342b9",
   "name": "Shopify",
   "dockerRepository": "airbyte/source-shopify",
-  "dockerImageTag": "0.1.8",
+  "dockerImageTag": "0.1.9",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/shopify"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -133,7 +133,7 @@
 - sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9
   name: Shopify
   dockerRepository: airbyte/source-shopify
-  dockerImageTag: 0.1.8
+  dockerImageTag: 0.1.9
   documentationUrl: https://docs.airbyte.io/integrations/sources/shopify
 - sourceDefinitionId: 9845d17a-45f1-4070-8a60-50914b1c8e2b
   name: HTTP Request

diff --git a/airbyte-integrations/connectors/source-shopify/Dockerfile b/airbyte-integrations/connectors/source-shopify/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.8
+LABEL io.airbyte.version=0.1.9
 LABEL io.airbyte.name=airbyte/source-shopify
diff --git a/airbyte-integrations/connectors/source-shopify/README.md b/airbyte-integrations/connectors/source-shopify/README.md
@@ -70,6 +70,7 @@ You can also build the connector image via Gradle:
 ```
 ./gradlew :airbyte-integrations:connectors:source-shopify:airbyteDocker
 ```
+
 When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in
 the Dockerfile.
 
@@ -81,12 +82,14 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-shopify:dev check --co
 docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-shopify:dev discover --config /secrets/config.json
 docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-shopify:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
 ```
+
 ## Testing
 Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
 First install test dependencies into your virtual environment:
 ```
 pip install .[tests]
 ```
+
 ### Unit Tests
 To run unit tests locally, from the connector directory run:
 ```
@@ -100,6 +103,7 @@ Place custom tests inside `integration_tests/` folder, then, from the connector
 ```
 python3 -m pytest integration_tests
 ```
+
 #### Acceptance Tests
 Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information.
 If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
@@ -115,10 +119,12 @@ To run unit tests:
 ```
 ./gradlew :airbyte-integrations:connectors:source-shopify:unitTest
 ```
+
 To run acceptance and custom integration tests:
 ```
 ./gradlew :airbyte-integrations:connectors:source-shopify:integrationTest
 ```
+
 To build final build the connector:
 ```
 ./gradlew :airbyte-integrations:connectors:source-shopify:build

diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json
@@ -1,44 +1,44 @@
 {
+  "customers": {
+    "updated_at": "2020-06-08T21:09:26-07:00"
+  },
   "orders": {
-    "id": 2497723957406484
+    "updated_at": "2020-08-03T14:42:06-07:00"
   },
-  "collects": {
-    "id": 1727945110336789
+  "draft_orders": {
+    "updated_at": "2020-06-08T21:09:25-07:00"
   },
   "products": {
-    "id": 505931153421589
+    "updated_at": "2021-06-25T06:13:37-07:00"
   },
-  "customers": {
-    "id": 366477377551899
+  "abandoned_checkouts": {
+    "updated_at": "2021-06-29T02:14:14-07:00"
   },
   "metafields": {
-    "id": 1126916852947498
-  },
-  "transactions": {
-    "id": 312557948124698
+    "updated_at": "2019-11-03T21:15:00-08:00"
   },
-  "order_refunds": {
-    "id": 63997837325598
+  "collects": {
+    "id": 29523654213791
   },
   "custom_collections": {
-    "id": 15327628499498
+    "updated_at": "2021-06-29T09:41:33-07:00"
   },
-  "abandoned_checkouts": {
-    "id": 142548663666229
+  "order_refunds": {
+    "created_at": "2020-02-18T19:49:23-08:00"
   },
   "order_risks": {
-    "id": 593300439055898
+    "id": 5933004390559
+  },
+  "transactions": {
+    "created_at": "2020-06-08T21:09:20-07:00"
   },
   "pages": {
-    "id": 131092082366699
+    "updated_at": "2020-06-01T17:08:25-07:00"
   },
   "price_rules": {
-    "id": 935355809987999
+    "updated_at": "2021-06-29T06:35:08-07:00"
   },
   "discount_codes": {
-    "id": 980808098689434
-  },
-  "draft_orders": {
-    "id": 94068132465507
+    "updated_at": "2021-06-29T06:35:08-07:00"
   }
 }
diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json
@@ -201,10 +201,10 @@
         },
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["id"]
+        "default_cursor_field": ["updated_at"]
       },
       "sync_mode": "incremental",
-      "cursor_field": ["id"],
+      "cursor_field": ["updated_at"],
       "destination_sync_mode": "append"
     },
     {
@@ -1638,10 +1638,10 @@
         },
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["id"]
+        "default_cursor_field": ["updated_at"]
       },
       "sync_mode": "incremental",
-      "cursor_field": ["id"],
+      "cursor_field": ["updated_at"],
       "destination_sync_mode": "append"
     },
     {
@@ -2335,10 +2335,10 @@
         },
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["id"]
+        "default_cursor_field": ["updated_at"]
       },
       "sync_mode": "incremental",
-      "cursor_field": ["id"],
+      "cursor_field": ["updated_at"],
       "destination_sync_mode": "append"
     },
     {
@@ -2592,10 +2592,10 @@
         },
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["id"]
+        "default_cursor_field": ["updated_at"]
       },
       "sync_mode": "incremental",
-      "cursor_field": ["id"],
+      "cursor_field": ["updated_at"],
       "destination_sync_mode": "append"
     },
     {
@@ -3373,10 +3373,10 @@
         },
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["id"]
+        "default_cursor_field": ["updated_at"]
       },
       "sync_mode": "incremental",
-      "cursor_field": ["id"],
+      "cursor_field": ["updated_at"],
       "destination_sync_mode": "append"
     },
     {
@@ -3425,6 +3425,45 @@
         },
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
+        "default_cursor_field": ["updated_at"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["updated_at"],
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "collects",
+        "json_schema": {
+          "type": "object",
+          "properties": {
+            "id": {
+              "type": ["null", "integer"]
+            },
+            "collection_id": {
+              "type": ["null", "integer"]
+            },
+            "created_at": {
+              "type": ["null", "string"],
+              "format": "date-time"
+            },
+            "position": {
+              "type": ["null", "integer"]
+            },
+            "product_id": {
+              "type": ["null", "integer"]
+            },
+            "sort_value": {
+              "type": ["null", "string"]
+            },
+            "updated_at": {
+              "type": ["null", "string"],
+              "format": "date-time"
+            }
+          }
+        },
+        "supported_sync_modes": ["incremental", "full_refresh"],
+        "source_defined_cursor": true,
         "default_cursor_field": ["id"]
       },
       "sync_mode": "incremental",
@@ -3491,10 +3530,10 @@
         },
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["id"]
+        "default_cursor_field": ["updated_at"]
       },
       "sync_mode": "incremental",
-      "cursor_field": ["id"],
+      "cursor_field": ["updated_at"],
       "destination_sync_mode": "append"
     },
     {
@@ -3897,10 +3936,10 @@
         },
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["id"]
+        "default_cursor_field": ["created_at"]
       },
       "sync_mode": "incremental",
-      "cursor_field": ["id"],
+      "cursor_field": ["created_at"],
       "destination_sync_mode": "append"
     },
     {
@@ -4054,10 +4093,10 @@
         },
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["id"]
+        "default_cursor_field": ["created_at"]
       },
       "sync_mode": "incremental",
-      "cursor_field": ["id"],
+      "cursor_field": ["created_at"],
       "destination_sync_mode": "append"
     },
     {
@@ -4103,10 +4142,10 @@
         },
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["id"]
+        "default_cursor_field": ["updated_at"]
       },
       "sync_mode": "incremental",
-      "cursor_field": ["id"],
+      "cursor_field": ["updated_at"],
       "destination_sync_mode": "append"
     },
     {
@@ -4265,10 +4304,10 @@
         },
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["id"]
+        "default_cursor_field": ["updated_at"]
       },
       "sync_mode": "incremental",
-      "cursor_field": ["id"],
+      "cursor_field": ["updated_at"],
       "destination_sync_mode": "append"
     },
     {
@@ -4301,10 +4340,10 @@
         },
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["id"]
+        "default_cursor_field": ["updated_at"]
       },
       "sync_mode": "incremental",
-      "cursor_field": ["id"],
+      "cursor_field": ["updated_at"],
       "destination_sync_mode": "append"
     }
   ]

diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/no_refunds_catalog.json b/airbyte-integrations/connectors/source-shopify/integration_tests/no_refunds_catalog.json
@@ -2490,10 +2490,10 @@
         },
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["id"]
+        "default_cursor_field": ["updated_at"]
       },
       "sync_mode": "incremental",
-      "cursor_field": ["id"],
+      "cursor_field": ["updated_at"],
       "destination_sync_mode": "append"
     },
     {
@@ -3586,6 +3586,45 @@
       "cursor_field": ["id"],
       "destination_sync_mode": "append"
     },
+    {
+      "stream": {
+        "name": "collects",
+        "json_schema": {
+          "type": "object",
+          "properties": {
+            "id": {
+              "type": ["null", "integer"]
+            },
+            "collection_id": {
+              "type": ["null", "integer"]
+            },
+            "created_at": {
+              "type": ["null", "string"],
+              "format": "date-time"
+            },
+            "position": {
+              "type": ["null", "integer"]
+            },
+            "product_id": {
+              "type": ["null", "integer"]
+            },
+            "sort_value": {
+              "type": ["null", "string"]
+            },
+            "updated_at": {
+              "type": ["null", "string"],
+              "format": "date-time"
+            }
+          }
+        },
+        "supported_sync_modes": ["incremental", "full_refresh"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["id"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["id"],
+      "destination_sync_mode": "append"
+    },
     {
       "stream": {
         "name": "custom_collections",