source-faker: cursor = updated_at & remove records_per_sync

airbyte-integrations/connectors/source-faker/Dockerfile

@@ -34,5 +34,5 @@ COPY source_faker ./source_faker
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=2.1.0
+LABEL io.airbyte.version=3.0.0
 LABEL io.airbyte.name=airbyte/source-faker

airbyte-integrations/connectors/source-faker/README.md

@@ -164,3 +164,5 @@ You've checked out the repo, implemented a million dollar feature, and you're re
 1. Create a Pull Request.
 1. Pat yourself on the back for being an awesome contributor.
 1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
+
+The end

airbyte-integrations/connectors/source-faker/acceptance-test-config.yml

@@ -14,16 +14,48 @@ acceptance_tests:
     tests:
       - config_path: secrets/config.json
         backward_compatibility_tests_config:
-          disable_for_version: "1.0.0" # We changed the cursor field of the Purchases stream in 2.0.0
+          disable_for_version: "2.1.0" # We changed the cursor field of all streams to be "updated_at"
   basic_read:
     tests:
       - config_path: secrets/config.json
         expect_records:
           path: integration_tests/expected_records.jsonl
+        ignored_fields:
+          users:
+            - name: updated_at
+              bypass_reason: "dynamic field"
+            - name: created_at
+              bypass_reason: "dynamic field"
+          products:
+            - name: updated_at
+              bypass_reason: "dynamic field"
+            - name: created_at
+              bypass_reason: "dynamic field"
+          purchases:
+            - name: updated_at
+              bypass_reason: "dynamic field"
+            - name: created_at
+              bypass_reason: "dynamic field"
   full_refresh:
     tests:
       - config_path: secrets/config.json
         configured_catalog_path: integration_tests/configured_catalog.json
+        ignored_fields:
+          users:
+            - name: updated_at
+              bypass_reason: "dynamic field"
+            - name: created_at
+              bypass_reason: "dynamic field"
+          products:
+            - name: updated_at
+              bypass_reason: "dynamic field"
+            - name: created_at
+              bypass_reason: "dynamic field"
+          purchases:
+            - name: updated_at
+              bypass_reason: "dynamic field"
+            - name: created_at
+              bypass_reason: "dynamic field"
   incremental:
     tests:
       - config_path: secrets/config.json

airbyte-integrations/connectors/source-faker/csv_export/README.md

@@ -21,14 +21,13 @@ But let's assume we don't have 1TB of local hard disk. So, we want to make 10 ch
 ```json
 {
   "count": 2039840637,
-  "seed": 0,
-  "records_per_sync": 203984064
+  "seed": 0
 }
 ```
 
 **`state.json`**
 
-At the end of every sync, increment the `id` in the users stream and the `user_id` in the purchases stream by `203984064`, the `records_per_sync` chunk size
+At the end of every sync, increment the `id` in the users stream and the `user_id` in the purchases stream by `203984064`
 
 ```json
 [

airbyte-integrations/connectors/source-faker/integration_tests/abnormal_state.json

@@ -3,7 +3,7 @@
     "type": "STREAM",
     "stream": {
       "stream_state": {
-        "id": 11
+        "updated_at": 11
       },
       "stream_descriptor": {
         "name": "users"
@@ -14,7 +14,7 @@
     "type": "STREAM",
     "stream": {
       "stream_state": {
-        "user_id": 11
+        "updated_at": 11
       },
       "stream_descriptor": {
         "name": "purchases"
@@ -25,7 +25,7 @@
     "type": "STREAM",
     "stream": {
       "stream_state": {
-        "id": 101
+        "updated_at": 101
       },
       "stream_descriptor": {
         "name": "products"

airbyte-integrations/connectors/source-faker/integration_tests/configured_catalog.json

@@ -48,6 +48,16 @@
             "id": { "type": "number" },
             "user_id": { "type": "number" },
             "product_id": { "type": "number" },
+            "created_at": {
+              "type": "string",
+              "format": "date-time",
+              "airbyte_type": "timestamp_with_timezone"
+            },
+            "updated_at": {
+              "type": "string",
+              "format": "date-time",
+              "airbyte_type": "timestamp_with_timezone"
+            },
             "added_to_cart_at": {
               "type": ["null", "string"],
               "format": "date-time",
@@ -86,10 +96,15 @@
               "type": "string",
               "format": "date-time",
               "airbyte_type": "timestamp_with_timezone"
+            },
+            "updated_at": {
+              "type": "string",
+              "format": "date-time",
+              "airbyte_type": "timestamp_with_timezone"
             }
           }
         },
-        "supported_sync_modes": ["incremental"],
+        "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
         "default_cursor_field": ["created_at"]
       },

airbyte-integrations/connectors/source-faker/integration_tests/sample_config.json

@@ -1,6 +1,6 @@
 {
   "count": 10,
   "seed": 0,
-  "records_per_sync": 10,
-  "parallelism": 1
+  "parallelism": 1,
+  "always_updated": false
 }

airbyte-integrations/connectors/source-faker/metadata.yaml

@@ -4,7 +4,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: dfd88b22-b603-4c3d-aad7-3701784586b1
-  dockerImageTag: 2.1.0
+  dockerImageTag: 3.0.0
   dockerRepository: airbyte/source-faker
   githubIssueLabel: source-faker
   icon: faker.svg

airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py

@@ -72,6 +72,7 @@ def generate(self, user_id: int) -> List[Dict]:
             id = user_id + i + 1 - id_offset
             time_a = dt.datetime()
             time_b = dt.datetime()
+            updated_at = format_airbyte_time(datetime.datetime.now())
             created_at = time_a if time_a <= time_b else time_b
             product_id = numeric.integer_number(1, total_products)
             added_to_cart_at = self.random_date_in_range(created_at)
@@ -88,6 +89,8 @@ def generate(self, user_id: int) -> List[Dict]:
                 "id": id,
                 "product_id": product_id,
                 "user_id": user_id + 1,
+                "created_at": created_at,
+                "updated_at": updated_at,
                 "added_to_cart_at": format_airbyte_time(added_to_cart_at) if added_to_cart_at is not None else None,
                 "purchased_at": format_airbyte_time(purchased_at) if purchased_at is not None else None,
                 "returned_at": format_airbyte_time(returned_at) if returned_at is not None else None,

airbyte-integrations/connectors/source-faker/source_faker/schemas/products.json

@@ -11,6 +11,11 @@
       "type": "string",
       "format": "date-time",
       "airbyte_type": "timestamp_with_timezone"
+    },
+    "updated_at": {
+      "type": "string",
+      "format": "date-time",
+      "airbyte_type": "timestamp_with_timezone"
     }
   }
 }

airbyte-integrations/connectors/source-faker/source_faker/schemas/purchases.json

@@ -5,6 +5,16 @@
     "id": { "type": "number" },
     "user_id": { "type": "number" },
     "product_id": { "type": "number" },
+    "created_at": {
+      "type": "string",
+      "format": "date-time",
+      "airbyte_type": "timestamp_with_timezone"
+    },
+    "updated_at": {
+      "type": "string",
+      "format": "date-time",
+      "airbyte_type": "timestamp_with_timezone"
+    },
     "added_to_cart_at": {
       "type": ["null", "string"],
       "format": "date-time",

airbyte-integrations/connectors/source-faker/source_faker/source.py

@@ -21,12 +21,12 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) ->
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         count: int = config["count"] if "count" in config else 0
         seed: int = config["seed"] if "seed" in config else None
-        records_per_sync: int = config["records_per_sync"] if "records_per_sync" in config else 500
         records_per_slice: int = config["records_per_slice"] if "records_per_slice" in config else 100
+        always_updated: bool = config["always_updated"] if "always_updated" in config else True
         parallelism: int = config["parallelism"] if "parallelism" in config else 4
 
         return [
-            Products(count, seed, parallelism, records_per_sync, records_per_slice),
-            Users(count, seed, parallelism, records_per_sync, records_per_slice),
-            Purchases(count, seed, parallelism, records_per_sync, records_per_slice),
+            Products(count, seed, parallelism, records_per_slice, always_updated),
+            Users(count, seed, parallelism, records_per_slice, always_updated),
+            Purchases(count, seed, parallelism, records_per_slice, always_updated),
         ]

airbyte-integrations/connectors/source-faker/source_faker/spec.json

@@ -22,21 +22,19 @@
         "default": -1,
         "order": 1
       },
-      "records_per_sync": {
-        "title": "Records Per Sync",
-        "description": "How many fake records will be returned for each sync, for each stream?  By default, it will take 2 syncs to create the requested 1000 records.",
-        "type": "integer",
-        "minimum": 1,
-        "default": 500,
-        "order": 2
-      },
       "records_per_slice": {
         "title": "Records Per Stream Slice",
         "description": "How many fake records will be in each page (stream slice), before a state message is emitted?",
         "type": "integer",
         "minimum": 1,
         "default": 1000,
-        "order": 3
+        "order": 2
+      },
+      "always_updated": {
+        "title": "Always Updated",
+        "description": "Should the updated_at values for every record be new each sync?",
+        "type": "boolean",
+        "default": true
       },
       "parallelism": {
         "title": "Parallelism",