diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 26c4c500ed4ab6..7a56685403a478 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -467,7 +467,7 @@ - name: Sample Data (Faker) sourceDefinitionId: dfd88b22-b603-4c3d-aad7-3701784586b1 dockerRepository: airbyte/source-faker - dockerImageTag: 1.0.0 + dockerImageTag: 2.0.0 documentationUrl: https://docs.airbyte.com/integrations/sources/faker sourceType: api releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index e45e96420691e5..f8efa0921ef66c 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3866,7 +3866,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-faker:1.0.0" +- dockerImage: "airbyte/source-faker:2.0.0" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/faker" connectionSpecification: @@ -3907,8 +3907,17 @@ \ before a state message is emitted?" type: "integer" minimum: 1 - default: 100 + default: 1000 order: 3 + parallelism: + title: "Parallelism" + description: "How many parallel workers should we use to generate fake data?\ + \ Choose a value equal to the number of CPUs you will allocate to this\ + \ source." + type: "integer" + minimum: 1 + default: 4 + order: 4 supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] diff --git a/airbyte-integrations/connectors/source-faker/Dockerfile b/airbyte-integrations/connectors/source-faker/Dockerfile index 17f51a82818d81..1e2a25e003d3cf 100644 --- a/airbyte-integrations/connectors/source-faker/Dockerfile +++ b/airbyte-integrations/connectors/source-faker/Dockerfile @@ -34,5 +34,5 @@ COPY source_faker ./source_faker ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=2.0.0 LABEL io.airbyte.name=airbyte/source-faker diff --git a/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml b/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml index f6aed27fd225db..f9eaae62331b7f 100644 --- a/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml @@ -14,7 +14,7 @@ acceptance_tests: tests: - config_path: secrets/config.json backward_compatibility_tests_config: - disable_for_version: "0.2.1" + disable_for_version: "1.0.0" # We changed the cursor field of the Purchases stream in 2.0.0 basic_read: tests: - config_path: secrets/config.json diff --git a/airbyte-integrations/connectors/source-faker/csv_export/README.md b/airbyte-integrations/connectors/source-faker/csv_export/README.md new file mode 100644 index 00000000000000..fdc5245ff308fe --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/csv_export/README.md @@ -0,0 +1,76 @@ +# Python Source CSV Export + +This collection of tools is used to run the source and capture it's AirbyteMessages and convert them into CSV files. This is useful if you want to manually inspect this data or load it into a database manually. + +To be fast, we make use of parallel processing per-stream and only using command-line tools. This works by the main file (`main.sh`) running the source via python and tee-ing the output of RECORDS to sub-scripts which use `jq` to convert the records into CSV-delimited output, which we finally write to disk. + +As we read the connector config files, e.g. `--config secrets/config.json --state secrets/state.json --catalog integration_tests/configured_catalog.json`, you can manually step forward your sync if you need to read and store the input in chunks. + +## The road to 1TB of faker data + +There's commentary on this at https://github.com/airbytehq/airbyte/pull/20558, along with some cool SQL tricks. + +- 2 Billion faker users for 1TB: `10,000,000*(1024/5.02) = 2,039,840,637` +- 200 Million faker users for 100GB: `10,000,000*(100/5.02) = 199,203,187` +- 20 Million faker users for 10GB: `10,000,000*(10/5.02) = 19,920,318` + +But let's assume we don't have 1TB of local hard disk. So, we want to make 10 chunks of data, each around 100GB in size. + +**`config.json`** + +```json +{ + "count": 2039840637, + "seed": 0, + "records_per_sync": 203984064 +} +``` + +**`state.json`** + +At the end of every sync, increment the `id` in the users stream and the `user_id` in the purchases stream by `203984064`, the `records_per_sync` chunk size + +```json +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 0 + }, + "stream_descriptor": { + "name": "users" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 0, + "user_id": 0 + }, + "stream_descriptor": { + "name": "purchases" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 0 + }, + "stream_descriptor": { + "name": "products" + } + } + } +] +``` + +Finally, ensure that you've opted-into all the streams in `integration_tests/configured_catalog.json` + +## TODO + +- This is currently set up very manually, in that we build bash scripts for each stream and manually populate the header information. This information all already lives in the connector's catalog. We probably could build these bash files on-demand with a python script... diff --git a/airbyte-integrations/connectors/source-faker/csv_export/main.sh b/airbyte-integrations/connectors/source-faker/csv_export/main.sh new file mode 100755 index 00000000000000..c9cd903f302dbf --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/csv_export/main.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +cd "$(dirname "$0")" +cd ".." + +mkdir -p /tmp/csv + +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json --state secrets/state.json \ + | tee >(./csv_export/purchases.sh) >(./csv_export/products.sh) >(./csv_export/users.sh) diff --git a/airbyte-integrations/connectors/source-faker/csv_export/products.sh b/airbyte-integrations/connectors/source-faker/csv_export/products.sh new file mode 100755 index 00000000000000..505340e7391967 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/csv_export/products.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +cd "$(dirname "$0")" + +FILE="/tmp/csv/products.csv" + +rm -rf $FILE + +echo "make, model, price, created_at" >> $FILE + +jq -c 'select((.type | contains("RECORD")) and (.record.stream | contains("products"))) .record.data' \ + | jq -r '[.make, .model, .price, .created_at] | @csv' \ + >> $FILE diff --git a/airbyte-integrations/connectors/source-faker/csv_export/purchases.sh b/airbyte-integrations/connectors/source-faker/csv_export/purchases.sh new file mode 100755 index 00000000000000..9a79e225fdebcb --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/csv_export/purchases.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +cd "$(dirname "$0")" + +FILE="/tmp/csv/purchases.csv" + +rm -rf $FILE + +echo "id, product_id, user_id, added_to_cart_at, purchased_at, returned_at" >> $FILE + +jq -c 'select((.type | contains("RECORD")) and (.record.stream | contains("purchases"))) .record.data' \ + | jq -r '[.id, .product_id, .user_id, .added_to_cart_at, .purchased_at, .returned_at] | @csv' \ + >> $FILE diff --git a/airbyte-integrations/connectors/source-faker/csv_export/users.sh b/airbyte-integrations/connectors/source-faker/csv_export/users.sh new file mode 100755 index 00000000000000..a6ac8f374b1da5 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/csv_export/users.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +cd "$(dirname "$0")" + +FILE="/tmp/csv/users.csv" + +rm -rf $FILE + +echo "id, created_at, updated_at, name, title, age, email, telephone, gender, language, academic_degree, nationality, occupation, height, blood_type, weight" >> $FILE + +jq -c 'select((.type | contains("RECORD")) and (.record.stream | contains("users"))) .record.data' \ + | jq -r '[.id, .created_at, .updated_at, .name, .title, .age, .email, .telephone, .gender, .language, .academic_degree, .nationality, .occupation, .height, .blood_type, .weight] | @csv' \ + >> $FILE diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl index 5ade6ae247dff2..685c3769f6a41d 100644 --- a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl @@ -1,123 +1,120 @@ -{"stream": "users", "data": {"id": 1, "created_at": "2009-08-12T18:57:58+00:00", "updated_at": "2012-07-02T08:32:31+00:00", "name": "Reda", "title": "M.Sc.Tech.", "age": 47, "email": "locations1983@protonmail.com", "telephone": "+1-(110)-795-7610", "gender": "Male", "language": "Tamil", "academic_degree": "Master", "nationality": "Italian", "occupation": "Word Processing Operator", "height": "1.55", "blood_type": "B\u2212", "weight": 58}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 2, "created_at": "2008-09-23T19:57:09+00:00", "updated_at": "2016-03-10T04:48:06+00:00", "name": "Tristan", "title": "M.Sc.Tech.", "age": 32, "email": "variations1847@duck.com", "telephone": "683-770-9281", "gender": "Other", "language": "Bosnian", "academic_degree": "Bachelor", "nationality": "Estonian", "occupation": "Tiler", "height": "2.00", "blood_type": "AB\u2212", "weight": 44}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 3, "created_at": "2003-06-14T10:39:40+00:00", "updated_at": "2003-12-03T21:21:30+00:00", "name": "Yuki", "title": "Miss", "age": 50, "email": "vacuum2027@yahoo.com", "telephone": "1-321-809-2061", "gender": "Female", "language": "Armenian", "academic_degree": "Bachelor", "nationality": "Swiss", "occupation": "Valuer", "height": "1.84", "blood_type": "O\u2212", "weight": 71}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 4, "created_at": "2001-09-30T00:05:46+00:00", "updated_at": "2006-09-16T14:55:33+00:00", "name": "Fred", "title": "MMath", "age": 47, "email": "causes1859@outlook.com", "telephone": "(827) 127-3811", "gender": "Female", "language": "Assamese", "academic_degree": "PhD", "nationality": "Russian", "occupation": "Turkey Farmer", "height": "1.80", "blood_type": "A+", "weight": 39}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 5, "created_at": "2012-12-27T21:40:00+00:00", "updated_at": "2015-06-08T23:20:45+00:00", "name": "Emmitt", "title": "DPhil", "age": 39, "email": "athens1899@gmail.com", "telephone": "(470) 656-8003", "gender": "Other", "language": "English", "academic_degree": "Bachelor", "nationality": "Jordanian", "occupation": "Stone Sawyer", "height": "1.52", "blood_type": "A+", "weight": 82}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 6, "created_at": "2002-04-30T18:14:15+00:00", "updated_at": "2004-09-15T02:05:20+00:00", "name": "Hollis", "title": "MSc", "age": 52, "email": "fisheries1881@yandex.com", "telephone": "(519) 606-9896", "gender": "Other", "language": "Swati", "academic_degree": "Master", "nationality": "Chilean", "occupation": "Writer", "height": "1.85", "blood_type": "AB\u2212", "weight": 85}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 7, "created_at": "2003-09-11T17:13:51+00:00", "updated_at": "2016-08-04T09:35:18+00:00", "name": "Kip", "title": "M.D.", "age": 31, "email": "numbers1983@example.com", "telephone": "346-013-2638", "gender": "Other", "language": "Armenian", "academic_degree": "Master", "nationality": "Swiss", "occupation": "Salesman", "height": "1.89", "blood_type": "O+", "weight": 48}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 8, "created_at": "2012-06-19T07:18:11+00:00", "updated_at": "2017-10-10T14:05:38+00:00", "name": "Carie", "title": "Madam", "age": 49, "email": "watershed1819@example.com", "telephone": "(348) 881-9607", "gender": "Male", "language": "Kyrgyz", "academic_degree": "PhD", "nationality": "Guatemalan", "occupation": "Park Ranger", "height": "1.50", "blood_type": "O+", "weight": 83}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 9, "created_at": "2002-11-25T04:56:09+00:00", "updated_at": "2005-01-20T21:16:30+00:00", "name": "Steven", "title": "Mr.", "age": 54, "email": "llp1893@yahoo.com", "telephone": "830.247.8156", "gender": "Fluid", "language": "Catalan", "academic_degree": "Bachelor", "nationality": "Egyptian", "occupation": "Ambulance Driver", "height": "1.52", "blood_type": "AB+", "weight": 81}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 10, "created_at": "2001-02-23T17:43:25+00:00", "updated_at": "2022-09-09T16:51:15+00:00", "name": "Lore", "title": "Madam", "age": 61, "email": "resident2075@example.com", "telephone": "321.233.0702", "gender": "Female", "language": "Polish", "academic_degree": "Master", "nationality": "French", "occupation": "Registrar", "height": "1.99", "blood_type": "B+", "weight": 56}, "emitted_at": 1669830193008} -{"stream": "purchases", "data": {"id": 1, "product_id": 98, "user_id": 1, "added_to_cart_at": "2019-01-17T18:57:58+00:00", "purchased_at": "2020-06-30T18:57:58+00:00", "returned_at": null}, "emitted_at": 1669830193009} -{"stream": "purchases", "data": {"id": 2, "product_id": 39, "user_id": 2, "added_to_cart_at": "2019-06-02T19:57:09+00:00", "purchased_at": "2022-09-08T19:57:09+00:00", "returned_at": null}, "emitted_at": 1669830193009} -{"stream": "purchases", "data": {"id": 3, "product_id": 37, "user_id": 3, "added_to_cart_at": "2006-08-01T10:39:40+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1669830193009} -{"stream": "purchases", "data": {"id": 4, "product_id": 80, "user_id": 3, "added_to_cart_at": "2021-05-18T10:39:40+00:00", "purchased_at": "2022-11-14T10:39:40+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 5, "product_id": 40, "user_id": 4, "added_to_cart_at": "2003-12-18T00:05:46+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 6, "product_id": 88, "user_id": 4, "added_to_cart_at": "2009-02-24T00:05:46+00:00", "purchased_at": "2021-09-14T00:05:46+00:00", "returned_at": "2022-03-14T00:05:46+00:00"}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 7, "product_id": 79, "user_id": 5, "added_to_cart_at": "2020-03-03T21:40:00+00:00", "purchased_at": "2022-11-17T21:40:00+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 8, "product_id": 67, "user_id": 6, "added_to_cart_at": "2008-03-02T18:14:15+00:00", "purchased_at": "2020-06-21T18:14:15+00:00", "returned_at": "2020-09-24T18:14:15+00:00"}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 9, "product_id": 91, "user_id": 7, "added_to_cart_at": "2022-03-12T17:13:51+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 10, "product_id": 79, "user_id": 8, "added_to_cart_at": "2017-12-31T07:18:11+00:00", "purchased_at": "2019-05-14T07:18:11+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 11, "product_id": 91, "user_id": 8, "added_to_cart_at": "2022-03-24T07:18:11+00:00", "purchased_at": "2022-05-11T07:18:11+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 12, "product_id": 19, "user_id": 9, "added_to_cart_at": "2020-11-29T04:56:09+00:00", "purchased_at": "2022-03-02T04:56:09+00:00", "returned_at": "2022-04-12T04:56:09+00:00"}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 13, "product_id": 63, "user_id": 10, "added_to_cart_at": "2003-08-05T17:43:25+00:00", "purchased_at": "2015-12-15T17:43:25+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "products", "data": {"id": 1, "make": "Mazda", "model": "MX-5", "year": 2008, "price": 2869, "created_at": "2022-02-01T17:02:19+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 2, "make": "Mercedes-Benz", "model": "C-Class", "year": 2009, "price": 42397, "created_at": "2021-01-25T14:31:33+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 3, "make": "Honda", "model": "Accord Crosstour", "year": 2011, "price": 63293, "created_at": "2021-02-11T05:36:03+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 4, "make": "GMC", "model": "Jimmy", "year": 1998, "price": 34079, "created_at": "2022-01-24T03:00:03+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 5, "make": "Infiniti", "model": "FX", "year": 2004, "price": 17036, "created_at": "2021-10-02T03:55:44+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 6, "make": "Dodge", "model": "Intrepid", "year": 2002, "price": 65498, "created_at": "2022-01-18T00:41:08+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 7, "make": "Nissan", "model": "Frontier", "year": 2005, "price": 14516, "created_at": "2021-04-22T16:37:44+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 8, "make": "Chevrolet", "model": "Express 1500", "year": 2007, "price": 13023, "created_at": "2021-07-12T07:13:04+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 9, "make": "Bentley", "model": "Continental GTC", "year": 2008, "price": 43458, "created_at": "2021-03-17T05:43:15+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 10, "make": "Cadillac", "model": "DTS", "year": 2008, "price": 43859, "created_at": "2021-08-12T07:33:58+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 11, "make": "Dodge", "model": "Ram 2500", "year": 2000, "price": 82904, "created_at": "2021-09-03T10:51:16+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 12, "make": "Suzuki", "model": "SJ 410", "year": 1984, "price": 38667, "created_at": "2021-01-11T00:15:46+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 13, "make": "Audi", "model": "S4", "year": 2005, "price": 2391, "created_at": "2021-09-06T03:31:10+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 14, "make": "Chevrolet", "model": "Suburban 2500", "year": 1998, "price": 55733, "created_at": "2021-10-18T17:26:05+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 15, "make": "Ford", "model": "Ranger", "year": 2000, "price": 20228, "created_at": "2022-03-24T04:03:19+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 16, "make": "Chevrolet", "model": "Corvette", "year": 2009, "price": 75052, "created_at": "2021-12-31T03:38:21+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 17, "make": "Mitsubishi", "model": "Pajero", "year": 1993, "price": 84058, "created_at": "2021-10-15T00:25:34+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 18, "make": "Lincoln", "model": "LS", "year": 2002, "price": 34081, "created_at": "2022-02-14T22:12:01+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 19, "make": "Dodge", "model": "Magnum", "year": 2005, "price": 85545, "created_at": "2021-07-25T22:49:48+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 20, "make": "Pontiac", "model": "Grand Am", "year": 2001, "price": 54837, "created_at": "2021-10-15T14:08:30+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 21, "make": "Chevrolet", "model": "Suburban 1500", "year": 2006, "price": 89410, "created_at": "2021-03-23T15:40:43+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 22, "make": "GMC", "model": "Sierra 1500", "year": 2005, "price": 14288, "created_at": "2021-08-30T13:40:04+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 23, "make": "GMC", "model": "3500", "year": 1995, "price": 12011, "created_at": "2022-04-24T13:11:08+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 24, "make": "Mazda", "model": "Mazda5", "year": 2006, "price": 6393, "created_at": "2021-07-07T14:14:33+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 25, "make": "Chevrolet", "model": "Camaro", "year": 1967, "price": 71590, "created_at": "2021-01-10T21:50:22+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 26, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 23498, "created_at": "2022-04-20T00:52:20+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 27, "make": "Dodge", "model": "Caravan", "year": 1985, "price": 50071, "created_at": "2022-01-05T10:13:31+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 28, "make": "Nissan", "model": "240SX", "year": 1992, "price": 38379, "created_at": "2022-04-07T04:48:48+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 29, "make": "Oldsmobile", "model": "Intrigue", "year": 2002, "price": 21376, "created_at": "2021-10-01T13:30:49+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 30, "make": "Audi", "model": "TT", "year": 2011, "price": 40893, "created_at": "2021-02-28T23:06:37+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 31, "make": "Ford", "model": "Crown Victoria", "year": 2006, "price": 86225, "created_at": "2021-01-28T23:33:27+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 32, "make": "Toyota", "model": "Tacoma", "year": 2003, "price": 73558, "created_at": "2022-01-28T22:02:04+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 33, "make": "Buick", "model": "Regal", "year": 1994, "price": 32279, "created_at": "2022-04-04T13:35:49+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 34, "make": "Mercedes-Benz", "model": "C-Class", "year": 2001, "price": 98732, "created_at": "2021-03-30T23:16:05+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 35, "make": "GMC", "model": "Sierra 3500", "year": 2002, "price": 48267, "created_at": "2021-07-30T20:29:51+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 36, "make": "Pontiac", "model": "G6", "year": 2005, "price": 16766, "created_at": "2021-03-24T07:53:33+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 37, "make": "Subaru", "model": "Outback Sport", "year": 2002, "price": 34523, "created_at": "2021-12-23T22:47:32+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 38, "make": "Ferrari", "model": "F430", "year": 2007, "price": 31677, "created_at": "2021-01-11T04:49:57+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 39, "make": "Mitsubishi", "model": "Montero", "year": 2003, "price": 67136, "created_at": "2021-05-10T07:37:56+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 40, "make": "Nissan", "model": "Sentra", "year": 1993, "price": 78236, "created_at": "2021-11-10T23:48:26+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 41, "make": "Mitsubishi", "model": "3000GT", "year": 1993, "price": 58150, "created_at": "2021-09-08T06:55:22+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 42, "make": "Ford", "model": "E350", "year": 2012, "price": 55270, "created_at": "2021-03-24T13:17:37+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 43, "make": "Ford", "model": "Taurus", "year": 1987, "price": 13522, "created_at": "2021-10-27T21:03:59+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 44, "make": "Chevrolet", "model": "Avalanche", "year": 2012, "price": 9862, "created_at": "2021-07-13T12:22:26+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 45, "make": "Dodge", "model": "Charger", "year": 2012, "price": 81887, "created_at": "2021-04-24T01:48:24+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 46, "make": "Jaguar", "model": "S-Type", "year": 2005, "price": 34372, "created_at": "2021-04-03T08:56:17+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 47, "make": "Plymouth", "model": "Grand Voyager", "year": 1994, "price": 90637, "created_at": "2022-04-21T09:21:08+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 48, "make": "Pontiac", "model": "6000", "year": 1989, "price": 65165, "created_at": "2021-10-30T13:03:07+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 49, "make": "Lexus", "model": "IS", "year": 2006, "price": 22434, "created_at": "2021-01-16T10:45:52+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 50, "make": "Isuzu", "model": "VehiCROSS", "year": 2001, "price": 38180, "created_at": "2021-12-13T16:29:27+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 51, "make": "Buick", "model": "Regal", "year": 2000, "price": 38680, "created_at": "2021-12-29T22:25:54+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 52, "make": "Mercedes-Benz", "model": "E-Class", "year": 2007, "price": 51556, "created_at": "2021-07-06T11:42:23+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 53, "make": "Buick", "model": "LeSabre", "year": 2001, "price": 10904, "created_at": "2022-01-05T18:23:35+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 54, "make": "Porsche", "model": "928", "year": 1989, "price": 70917, "created_at": "2022-01-02T23:16:45+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 55, "make": "Lexus", "model": "RX", "year": 2007, "price": 5212, "created_at": "2021-07-10T15:02:53+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 56, "make": "Ford", "model": "Econoline E250", "year": 1996, "price": 75095, "created_at": "2021-02-04T16:17:18+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 57, "make": "Chevrolet", "model": "Blazer", "year": 2001, "price": 61918, "created_at": "2021-12-08T07:25:30+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 58, "make": "GMC", "model": "Savana 3500", "year": 2003, "price": 30307, "created_at": "2021-11-21T23:11:45+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 59, "make": "BMW", "model": "M", "year": 2002, "price": 24598, "created_at": "2021-05-28T04:08:53+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 60, "make": "Saturn", "model": "S-Series", "year": 1992, "price": 96288, "created_at": "2021-08-24T04:43:43+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 61, "make": "Chrysler", "model": "Sebring", "year": 2003, "price": 34753, "created_at": "2021-02-11T11:25:35+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 62, "make": "Lotus", "model": "Evora", "year": 2010, "price": 42760, "created_at": "2021-08-31T00:29:05+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 63, "make": "Jeep", "model": "Wrangler", "year": 2011, "price": 8684, "created_at": "2021-06-24T10:38:05+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 64, "make": "Ford", "model": "Expedition", "year": 2012, "price": 25653, "created_at": "2021-07-01T16:13:20+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 65, "make": "Chevrolet", "model": "Avalanche 2500", "year": 2006, "price": 3158, "created_at": "2021-08-14T10:55:13+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 66, "make": "Mazda", "model": "Mazda3", "year": 2012, "price": 79820, "created_at": "2021-05-25T21:55:52+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 67, "make": "Toyota", "model": "Tacoma", "year": 2005, "price": 73572, "created_at": "2021-01-22T09:56:02+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 68, "make": "Ford", "model": "Explorer Sport", "year": 2000, "price": 64579, "created_at": "2021-02-16T06:56:06+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 69, "make": "GMC", "model": "Savana Cargo Van", "year": 2006, "price": 65944, "created_at": "2021-09-12T14:08:53+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 70, "make": "Chevrolet", "model": "HHR", "year": 2009, "price": 8953, "created_at": "2021-08-17T04:25:43+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 71, "make": "Ford", "model": "Bronco II", "year": 1989, "price": 41811, "created_at": "2021-07-14T14:20:28+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 72, "make": "Chevrolet", "model": "Suburban 2500", "year": 2011, "price": 57488, "created_at": "2021-09-22T12:32:57+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 73, "make": "Suzuki", "model": "Grand Vitara", "year": 2008, "price": 6408, "created_at": "2021-11-12T23:19:52+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 74, "make": "Mazda", "model": "Mazda6", "year": 2012, "price": 14805, "created_at": "2021-06-01T01:55:32+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 75, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 33585, "created_at": "2022-01-09T04:28:54+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 76, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 2087, "created_at": "2022-03-28T00:28:16+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 77, "make": "Ford", "model": "F150", "year": 2007, "price": 17621, "created_at": "2021-03-23T15:08:10+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 78, "make": "Ford", "model": "Taurus", "year": 1995, "price": 16478, "created_at": "2021-06-07T22:29:50+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 79, "make": "Mitsubishi", "model": "Truck", "year": 1992, "price": 70616, "created_at": "2022-01-30T05:14:02+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 80, "make": "Dodge", "model": "Colt", "year": 1994, "price": 34163, "created_at": "2022-04-02T18:06:30+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 81, "make": "Mazda", "model": "RX-7", "year": 1991, "price": 29634, "created_at": "2021-01-06T10:30:59+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 82, "make": "Pontiac", "model": "Grand Prix", "year": 1984, "price": 88575, "created_at": "2021-02-24T06:06:57+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 83, "make": "Mazda", "model": "Mazdaspeed 3", "year": 2012, "price": 77723, "created_at": "2021-11-11T22:48:05+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 84, "make": "Alfa Romeo", "model": "Spider", "year": 1992, "price": 64288, "created_at": "2021-01-06T03:50:27+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 85, "make": "Audi", "model": "S8", "year": 2002, "price": 33718, "created_at": "2021-07-21T11:14:54+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 86, "make": "Isuzu", "model": "Amigo", "year": 1992, "price": 53335, "created_at": "2022-03-02T10:42:21+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 87, "make": "Toyota", "model": "Paseo", "year": 1996, "price": 74558, "created_at": "2021-10-02 14:54:58+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 88, "make": "Lincoln", "model": "Continental Mark VII", "year": 1986, "price": 42150, "created_at": "2021-10-02T04:48:53+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 89, "make": "Dodge", "model": "Dakota", "year": 1997, "price": 64516, "created_at": "2021-09-09T23:13:26+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 90, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 51461, "created_at": "2021-04-06T08:29:19+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 91, "make": "Pontiac", "model": "Vibe", "year": 2006, "price": 12134, "created_at": "2021-01-11T22:30:14+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 92, "make": "Volkswagen", "model": "Eos", "year": 2011, "price": 53128, "created_at": "2021-01-12T23:25:06+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 93, "make": "Mazda", "model": "Mazdaspeed6", "year": 2007, "price": 90902, "created_at": "2021-12-29T14:29:03+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 94, "make": "Nissan", "model": "Xterra", "year": 2005, "price": 41532, "created_at": "2021-09-07 09:00:49+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 95, "make": "Mercury", "model": "Sable", "year": 2005, "price": 71337, "created_at": "2021-01-31T22:13:44+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 96, "make": "BMW", "model": "330", "year": 2006, "price": 14494, "created_at": "2021-09-17T20:52:48+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 97, "make": "Audi", "model": "R8", "year": 2008, "price": 17642, "created_at": "2021-09-21T11:56:24+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 98, "make": "Cadillac", "model": "CTS-V", "year": 2007, "price": 19914, "created_at": "2021-09-02T15:38:46+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 99, "make": "GMC", "model": "1500 Club Coupe", "year": 1997, "price": 82288, "created_at": "2021-04-20T18:58:15+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 100, "make": "Buick", "model": "Somerset", "year": 1986, "price": 64148, "created_at": "2021-06-10T19:07:38+00:00"}, "emitted_at": 1669830193015} +{"stream": "users", "data": {"id": 1, "created_at": "2004-10-28T02:16:07+00:00", "updated_at": "2014-08-21T12:50:13+00:00", "name": "Rudolf", "title": "M.Des", "age": 66, "email": "wisconsin1930+1@yandex.com", "telephone": "(483) 676-2851", "gender": "Fluid", "language": "Arabic", "academic_degree": "Bachelor", "nationality": "Argentinian", "occupation": "Valve Technician", "height": "1.50", "blood_type": "B\u2212", "weight": 81}, "emitted_at": 1672699162606} +{"stream": "users", "data": {"id": 2, "created_at": "2000-12-15T08:46:51+00:00", "updated_at": "2015-01-29T12:27:38+00:00", "name": "Orville", "title": "Miss", "age": 30, "email": "recipes2070+2@yahoo.com", "telephone": "994.991.6727", "gender": "Other", "language": "Montenegrin", "academic_degree": "PhD", "nationality": "Costa Rican", "occupation": "Optical Advisor", "height": "1.64", "blood_type": "AB\u2212", "weight": 70}, "emitted_at": 1672699162606} +{"stream": "users", "data": {"id": 3, "created_at": "2017-01-31T12:43:13+00:00", "updated_at": "2018-02-11T00:01:01+00:00", "name": "Rachell", "title": "M.A.", "age": 21, "email": "assets1924+3@protonmail.com", "telephone": "+1-(118)-374-3865", "gender": "Female", "language": "Dutch", "academic_degree": "PhD", "nationality": "Danish", "occupation": "Aeronautical Engineer", "height": "1.89", "blood_type": "AB+", "weight": 63}, "emitted_at": 1672699162606} +{"stream": "users", "data": {"id": 4, "created_at": "2015-09-08T11:14:43+00:00", "updated_at": "2023-01-17T07:48:28+00:00", "name": "Yer", "title": "M.Sc.Tech.", "age": 24, "email": "necessary2035+4@example.org", "telephone": "+1-(294)-359-4840", "gender": "Fluid", "language": "Malay", "academic_degree": "PhD", "nationality": "Argentinian", "occupation": "Line Manager", "height": "1.82", "blood_type": "B+", "weight": 43}, "emitted_at": 1672699162607} +{"stream": "users", "data": {"id": 5, "created_at": "2014-05-30T00:26:53+00:00", "updated_at": "2017-11-04T05:40:46+00:00", "name": "Alton", "title": "Miss", "age": 31, "email": "implementing1836+5@example.org", "telephone": "(712) 129-6627", "gender": "Other", "language": "Maltese", "academic_degree": "Bachelor", "nationality": "Argentinian", "occupation": "Paint Consultant", "height": "1.69", "blood_type": "B\u2212", "weight": 88}, "emitted_at": 1672699162607} +{"stream": "users", "data": {"id": 6, "created_at": "2009-02-24T10:57:46+00:00", "updated_at": "2016-07-17T21:12:19+00:00", "name": "Octavio", "title": "Mr.", "age": 41, "email": "ind1929+6@yahoo.com", "telephone": "717-652-9752", "gender": "Male", "language": "Kurdish", "academic_degree": "PhD", "nationality": "Polish", "occupation": "Pathologist", "height": "1.83", "blood_type": "B\u2212", "weight": 41}, "emitted_at": 1672699162607} +{"stream": "users", "data": {"id": 7, "created_at": "2001-08-08T23:51:25+00:00", "updated_at": "2018-08-28T16:25:37+00:00", "name": "Casimira", "title": "B.Sc", "age": 63, "email": "coupled1824+7@live.com", "telephone": "1-515-852-9488", "gender": "Male", "language": "Khmer", "academic_degree": "Bachelor", "nationality": "Finnish", "occupation": "Chicken Chaser", "height": "1.60", "blood_type": "B\u2212", "weight": 75}, "emitted_at": 1672699162607} +{"stream": "users", "data": {"id": 8, "created_at": "2021-03-12T17:56:44+00:00", "updated_at": "2021-12-12T02:28:42+00:00", "name": "Terrell", "title": "Miss", "age": 55, "email": "stick1999+8@yahoo.com", "telephone": "023-973-2689", "gender": "Fluid", "language": "Tamil", "academic_degree": "PhD", "nationality": "Japanese", "occupation": "Trout Farmer", "height": "1.62", "blood_type": "O+", "weight": 43}, "emitted_at": 1672699162607} +{"stream": "users", "data": {"id": 9, "created_at": "2000-08-02T09:45:54+00:00", "updated_at": "2003-03-17T12:23:31+00:00", "name": "Ira", "title": "M.A.", "age": 37, "email": "efforts2075+9@yandex.com", "telephone": "(286) 981-5100", "gender": "Female", "language": "Croatian", "academic_degree": "Master", "nationality": "Cameroonian", "occupation": "Purchase Clerk", "height": "1.54", "blood_type": "AB+", "weight": 74}, "emitted_at": 1672699162607} +{"stream": "users", "data": {"id": 10, "created_at": "2007-01-25T06:34:58+00:00", "updated_at": "2018-10-13T20:10:10+00:00", "name": "Randall", "title": "Mrs.", "age": 34, "email": "intellectual1951+10@gmail.com", "telephone": "018-029-4112", "gender": "Male", "language": "Luxembourgish", "academic_degree": "Master", "nationality": "Mexican", "occupation": "Nursery Nurse", "height": "1.78", "blood_type": "AB\u2212", "weight": 58}, "emitted_at": 1672699162607} +{"stream": "purchases", "data": {"id": 1, "product_id": 8, "user_id": 1, "added_to_cart_at": "2003-02-23T11:53:10+00:00", "purchased_at": "2011-03-30T11:53:10+00:00", "returned_at": null}, "emitted_at": 1672699162839} +{"stream": "purchases", "data": {"id": 2, "product_id": 95, "user_id": 2, "added_to_cart_at": "2022-08-08T13:40:25+00:00", "purchased_at": "2022-10-11T13:40:25+00:00", "returned_at": null}, "emitted_at": 1672699162839} +{"stream": "purchases", "data": {"id": 3, "product_id": 28, "user_id": 3, "added_to_cart_at": "2021-09-06T00:23:29+00:00", "purchased_at": "2022-06-30T00:23:29+00:00", "returned_at": null}, "emitted_at": 1672699162839} +{"stream": "purchases", "data": {"id": 4, "product_id": 21, "user_id": 4, "added_to_cart_at": "2016-11-27T05:20:11+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1672699162839} +{"stream": "purchases", "data": {"id": 5, "product_id": 51, "user_id": 5, "added_to_cart_at": "2020-02-14T11:54:28+00:00", "purchased_at": "2022-08-13T11:54:28+00:00", "returned_at": null}, "emitted_at": 1672699162839} +{"stream": "purchases", "data": {"id": 6, "product_id": 35, "user_id": 6, "added_to_cart_at": "2013-05-20T14:41:33+00:00", "purchased_at": "2017-06-19T14:41:33+00:00", "returned_at": null}, "emitted_at": 1672699162839} +{"stream": "purchases", "data": {"id": 7, "product_id": 41, "user_id": 8, "added_to_cart_at": "2021-12-28T17:46:29+00:00", "purchased_at": "2022-08-01T17:46:29+00:00", "returned_at": null}, "emitted_at": 1672699162839} +{"stream": "purchases", "data": {"id": 8, "product_id": 22, "user_id": 9, "added_to_cart_at": "2022-07-15T15:19:19+00:00", "purchased_at": "2022-09-13T15:19:19+00:00", "returned_at": null}, "emitted_at": 1672699162840} +{"stream": "purchases", "data": {"id": 9, "product_id": 4, "user_id": 9, "added_to_cart_at": "2019-07-06T23:13:31+00:00", "purchased_at": "2020-06-25T23:13:31+00:00", "returned_at": null}, "emitted_at": 1672699162840} +{"stream": "purchases", "data": {"id": 10, "product_id": 66, "user_id": 10, "added_to_cart_at": "2017-08-09T02:50:52+00:00", "purchased_at": "2020-06-26T02:50:52+00:00", "returned_at": null}, "emitted_at": 1672699162840} +{"stream": "products", "data": {"id": 1, "make": "Mazda", "model": "MX-5", "year": 2008, "price": 2869, "created_at": "2022-02-01T17:02:19+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 2, "make": "Mercedes-Benz", "model": "C-Class", "year": 2009, "price": 42397, "created_at": "2021-01-25T14:31:33+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 3, "make": "Honda", "model": "Accord Crosstour", "year": 2011, "price": 63293, "created_at": "2021-02-11T05:36:03+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 4, "make": "GMC", "model": "Jimmy", "year": 1998, "price": 34079, "created_at": "2022-01-24T03:00:03+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 5, "make": "Infiniti", "model": "FX", "year": 2004, "price": 17036, "created_at": "2021-10-02T03:55:44+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 6, "make": "Dodge", "model": "Intrepid", "year": 2002, "price": 65498, "created_at": "2022-01-18T00:41:08+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 7, "make": "Nissan", "model": "Frontier", "year": 2005, "price": 14516, "created_at": "2021-04-22T16:37:44+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 8, "make": "Chevrolet", "model": "Express 1500", "year": 2007, "price": 13023, "created_at": "2021-07-12T07:13:04+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 9, "make": "Bentley", "model": "Continental GTC", "year": 2008, "price": 43458, "created_at": "2021-03-17T05:43:15+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 10, "make": "Cadillac", "model": "DTS", "year": 2008, "price": 43859, "created_at": "2021-08-12T07:33:58+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 11, "make": "Dodge", "model": "Ram 2500", "year": 2000, "price": 82904, "created_at": "2021-09-03T10:51:16+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 12, "make": "Suzuki", "model": "SJ 410", "year": 1984, "price": 38667, "created_at": "2021-01-11T00:15:46+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 13, "make": "Audi", "model": "S4", "year": 2005, "price": 2391, "created_at": "2021-09-06T03:31:10+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 14, "make": "Chevrolet", "model": "Suburban 2500", "year": 1998, "price": 55733, "created_at": "2021-10-18T17:26:05+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 15, "make": "Ford", "model": "Ranger", "year": 2000, "price": 20228, "created_at": "2022-03-24T04:03:19+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 16, "make": "Chevrolet", "model": "Corvette", "year": 2009, "price": 75052, "created_at": "2021-12-31T03:38:21+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 17, "make": "Mitsubishi", "model": "Pajero", "year": 1993, "price": 84058, "created_at": "2021-10-15T00:25:34+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 18, "make": "Lincoln", "model": "LS", "year": 2002, "price": 34081, "created_at": "2022-02-14T22:12:01+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 19, "make": "Dodge", "model": "Magnum", "year": 2005, "price": 85545, "created_at": "2021-07-25T22:49:48+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 20, "make": "Pontiac", "model": "Grand Am", "year": 2001, "price": 54837, "created_at": "2021-10-15T14:08:30+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 21, "make": "Chevrolet", "model": "Suburban 1500", "year": 2006, "price": 89410, "created_at": "2021-03-23T15:40:43+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 22, "make": "GMC", "model": "Sierra 1500", "year": 2005, "price": 14288, "created_at": "2021-08-30T13:40:04+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 23, "make": "GMC", "model": "3500", "year": 1995, "price": 12011, "created_at": "2022-04-24T13:11:08+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 24, "make": "Mazda", "model": "Mazda5", "year": 2006, "price": 6393, "created_at": "2021-07-07T14:14:33+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 25, "make": "Chevrolet", "model": "Camaro", "year": 1967, "price": 71590, "created_at": "2021-01-10T21:50:22+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 26, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 23498, "created_at": "2022-04-20T00:52:20+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 27, "make": "Dodge", "model": "Caravan", "year": 1985, "price": 50071, "created_at": "2022-01-05T10:13:31+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 28, "make": "Nissan", "model": "240SX", "year": 1992, "price": 38379, "created_at": "2022-04-07T04:48:48+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 29, "make": "Oldsmobile", "model": "Intrigue", "year": 2002, "price": 21376, "created_at": "2021-10-01T13:30:49+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 30, "make": "Audi", "model": "TT", "year": 2011, "price": 40893, "created_at": "2021-02-28T23:06:37+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 31, "make": "Ford", "model": "Crown Victoria", "year": 2006, "price": 86225, "created_at": "2021-01-28T23:33:27+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 32, "make": "Toyota", "model": "Tacoma", "year": 2003, "price": 73558, "created_at": "2022-01-28T22:02:04+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 33, "make": "Buick", "model": "Regal", "year": 1994, "price": 32279, "created_at": "2022-04-04T13:35:49+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 34, "make": "Mercedes-Benz", "model": "C-Class", "year": 2001, "price": 98732, "created_at": "2021-03-30T23:16:05+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 35, "make": "GMC", "model": "Sierra 3500", "year": 2002, "price": 48267, "created_at": "2021-07-30T20:29:51+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 36, "make": "Pontiac", "model": "G6", "year": 2005, "price": 16766, "created_at": "2021-03-24T07:53:33+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 37, "make": "Subaru", "model": "Outback Sport", "year": 2002, "price": 34523, "created_at": "2021-12-23T22:47:32+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 38, "make": "Ferrari", "model": "F430", "year": 2007, "price": 31677, "created_at": "2021-01-11T04:49:57+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 39, "make": "Mitsubishi", "model": "Montero", "year": 2003, "price": 67136, "created_at": "2021-05-10T07:37:56+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 40, "make": "Nissan", "model": "Sentra", "year": 1993, "price": 78236, "created_at": "2021-11-10T23:48:26+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 41, "make": "Mitsubishi", "model": "3000GT", "year": 1993, "price": 58150, "created_at": "2021-09-08T06:55:22+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 42, "make": "Ford", "model": "E350", "year": 2012, "price": 55270, "created_at": "2021-03-24T13:17:37+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 43, "make": "Ford", "model": "Taurus", "year": 1987, "price": 13522, "created_at": "2021-10-27T21:03:59+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 44, "make": "Chevrolet", "model": "Avalanche", "year": 2012, "price": 9862, "created_at": "2021-07-13T12:22:26+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 45, "make": "Dodge", "model": "Charger", "year": 2012, "price": 81887, "created_at": "2021-04-24T01:48:24+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 46, "make": "Jaguar", "model": "S-Type", "year": 2005, "price": 34372, "created_at": "2021-04-03T08:56:17+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 47, "make": "Plymouth", "model": "Grand Voyager", "year": 1994, "price": 90637, "created_at": "2022-04-21T09:21:08+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 48, "make": "Pontiac", "model": "6000", "year": 1989, "price": 65165, "created_at": "2021-10-30T13:03:07+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 49, "make": "Lexus", "model": "IS", "year": 2006, "price": 22434, "created_at": "2021-01-16T10:45:52+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 50, "make": "Isuzu", "model": "VehiCROSS", "year": 2001, "price": 38180, "created_at": "2021-12-13T16:29:27+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 51, "make": "Buick", "model": "Regal", "year": 2000, "price": 38680, "created_at": "2021-12-29T22:25:54+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 52, "make": "Mercedes-Benz", "model": "E-Class", "year": 2007, "price": 51556, "created_at": "2021-07-06T11:42:23+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 53, "make": "Buick", "model": "LeSabre", "year": 2001, "price": 10904, "created_at": "2022-01-05T18:23:35+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 54, "make": "Porsche", "model": "928", "year": 1989, "price": 70917, "created_at": "2022-01-02T23:16:45+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 55, "make": "Lexus", "model": "RX", "year": 2007, "price": 5212, "created_at": "2021-07-10T15:02:53+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 56, "make": "Ford", "model": "Econoline E250", "year": 1996, "price": 75095, "created_at": "2021-02-04T16:17:18+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 57, "make": "Chevrolet", "model": "Blazer", "year": 2001, "price": 61918, "created_at": "2021-12-08T07:25:30+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 58, "make": "GMC", "model": "Savana 3500", "year": 2003, "price": 30307, "created_at": "2021-11-21T23:11:45+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 59, "make": "BMW", "model": "M", "year": 2002, "price": 24598, "created_at": "2021-05-28T04:08:53+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 60, "make": "Saturn", "model": "S-Series", "year": 1992, "price": 96288, "created_at": "2021-08-24T04:43:43+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 61, "make": "Chrysler", "model": "Sebring", "year": 2003, "price": 34753, "created_at": "2021-02-11T11:25:35+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 62, "make": "Lotus", "model": "Evora", "year": 2010, "price": 42760, "created_at": "2021-08-31T00:29:05+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 63, "make": "Jeep", "model": "Wrangler", "year": 2011, "price": 8684, "created_at": "2021-06-24T10:38:05+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 64, "make": "Ford", "model": "Expedition", "year": 2012, "price": 25653, "created_at": "2021-07-01T16:13:20+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 65, "make": "Chevrolet", "model": "Avalanche 2500", "year": 2006, "price": 3158, "created_at": "2021-08-14T10:55:13+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 66, "make": "Mazda", "model": "Mazda3", "year": 2012, "price": 79820, "created_at": "2021-05-25T21:55:52+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 67, "make": "Toyota", "model": "Tacoma", "year": 2005, "price": 73572, "created_at": "2021-01-22T09:56:02+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 68, "make": "Ford", "model": "Explorer Sport", "year": 2000, "price": 64579, "created_at": "2021-02-16T06:56:06+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 69, "make": "GMC", "model": "Savana Cargo Van", "year": 2006, "price": 65944, "created_at": "2021-09-12T14:08:53+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 70, "make": "Chevrolet", "model": "HHR", "year": 2009, "price": 8953, "created_at": "2021-08-17T04:25:43+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 71, "make": "Ford", "model": "Bronco II", "year": 1989, "price": 41811, "created_at": "2021-07-14T14:20:28+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 72, "make": "Chevrolet", "model": "Suburban 2500", "year": 2011, "price": 57488, "created_at": "2021-09-22T12:32:57+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 73, "make": "Suzuki", "model": "Grand Vitara", "year": 2008, "price": 6408, "created_at": "2021-11-12T23:19:52+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 74, "make": "Mazda", "model": "Mazda6", "year": 2012, "price": 14805, "created_at": "2021-06-01T01:55:32+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 75, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 33585, "created_at": "2022-01-09T04:28:54+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 76, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 2087, "created_at": "2022-03-28T00:28:16+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 77, "make": "Ford", "model": "F150", "year": 2007, "price": 17621, "created_at": "2021-03-23T15:08:10+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 78, "make": "Ford", "model": "Taurus", "year": 1995, "price": 16478, "created_at": "2021-06-07T22:29:50+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 79, "make": "Mitsubishi", "model": "Truck", "year": 1992, "price": 70616, "created_at": "2022-01-30T05:14:02+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 80, "make": "Dodge", "model": "Colt", "year": 1994, "price": 34163, "created_at": "2022-04-02T18:06:30+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 81, "make": "Mazda", "model": "RX-7", "year": 1991, "price": 29634, "created_at": "2021-01-06T10:30:59+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 82, "make": "Pontiac", "model": "Grand Prix", "year": 1984, "price": 88575, "created_at": "2021-02-24T06:06:57+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 83, "make": "Mazda", "model": "Mazdaspeed 3", "year": 2012, "price": 77723, "created_at": "2021-11-11T22:48:05+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 84, "make": "Alfa Romeo", "model": "Spider", "year": 1992, "price": 64288, "created_at": "2021-01-06T03:50:27+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 85, "make": "Audi", "model": "S8", "year": 2002, "price": 33718, "created_at": "2021-07-21T11:14:54+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 86, "make": "Isuzu", "model": "Amigo", "year": 1992, "price": 53335, "created_at": "2022-03-02T10:42:21+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 87, "make": "Toyota", "model": "Paseo", "year": 1996, "price": 74558, "created_at": "2021-10-02 14:54:58+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 88, "make": "Lincoln", "model": "Continental Mark VII", "year": 1986, "price": 42150, "created_at": "2021-10-02T04:48:53+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 89, "make": "Dodge", "model": "Dakota", "year": 1997, "price": 64516, "created_at": "2021-09-09T23:13:26+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 90, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 51461, "created_at": "2021-04-06T08:29:19+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 91, "make": "Pontiac", "model": "Vibe", "year": 2006, "price": 12134, "created_at": "2021-01-11T22:30:14+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 92, "make": "Volkswagen", "model": "Eos", "year": 2011, "price": 53128, "created_at": "2021-01-12T23:25:06+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 93, "make": "Mazda", "model": "Mazdaspeed6", "year": 2007, "price": 90902, "created_at": "2021-12-29T14:29:03+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 94, "make": "Nissan", "model": "Xterra", "year": 2005, "price": 41532, "created_at": "2021-09-07 09:00:49+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 95, "make": "Mercury", "model": "Sable", "year": 2005, "price": 71337, "created_at": "2021-01-31T22:13:44+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 96, "make": "BMW", "model": "330", "year": 2006, "price": 14494, "created_at": "2021-09-17T20:52:48+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 97, "make": "Audi", "model": "R8", "year": 2008, "price": 17642, "created_at": "2021-09-21T11:56:24+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 98, "make": "Cadillac", "model": "CTS-V", "year": 2007, "price": 19914, "created_at": "2021-09-02T15:38:46+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 99, "make": "GMC", "model": "1500 Club Coupe", "year": 1997, "price": 82288, "created_at": "2021-04-20T18:58:15+00:00"}, "emitted_at": 1672699162847} +{"stream": "products", "data": {"id": 100, "make": "Buick", "model": "Somerset", "year": 1986, "price": 64148, "created_at": "2021-06-10T19:07:38+00:00"}, "emitted_at": 1672699162847} diff --git a/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py b/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py new file mode 100644 index 00000000000000..be338ddb4afe0e --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.models import AirbyteMessage + + +class AirbyteMessageWithCachedJSON(AirbyteMessage): + """ + I a monkeypatch to AirbyteMessage which pre-renders the JSON-representation of the object upon initialization. + This allows the JSON to be calculated in the process that builds the object rather than the main process. + + Note: We can't use @cache here because the LRU cache is not serializable when passed to child workers. + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._json = self.json(exclude_unset=True) + self.json = self.get_json + + def get_json(self, **kwargs): + return self._json diff --git a/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py new file mode 100644 index 00000000000000..53cf05be417794 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py @@ -0,0 +1,103 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import datetime +from multiprocessing import current_process +from typing import Dict + +from airbyte_cdk.models import AirbyteRecordMessage, Type +from mimesis import Datetime, Numeric + +from .airbyte_message_with_cached_json import AirbyteMessageWithCachedJSON +from .utils import format_airbyte_time, now_millis + + +class PurchaseGenerator: + def __init__(self, stream_name: str, seed: int) -> None: + self.stream_name = stream_name + self.seed = seed + + def prepare(self): + """ + Note: the instances of the mimesis generators need to be global. + Yes, they *should* be able to be instance variables on this class, which should only instantiated once-per-worker, but that's not quite the case: + * relying only on prepare as a pool initializer fails because we are calling the parent process's method, not the fork + * Calling prepare() as part of generate() (perhaps checking if self.person is set) and then `print(self, current_process()._identity, current_process().pid)` reveals multiple object IDs in the same process, resetting the internal random counters + """ + + seed_with_offset = self.seed + if self.seed is not None and len(current_process()._identity) > 0: + seed_with_offset = self.seed + current_process()._identity[0] + + global dt + global numeric + + dt = Datetime(seed=seed_with_offset) + numeric = Numeric(seed=seed_with_offset) + + def random_date_in_range( + self, start_date: datetime.datetime, end_date: datetime.datetime = datetime.datetime.now() + ) -> datetime.datetime: + time_between_dates = end_date - start_date + days_between_dates = time_between_dates.days + if days_between_dates < 2: + days_between_dates = 2 + random_number_of_days = numeric.integer_number(0, days_between_dates) + random_date = start_date + datetime.timedelta(days=random_number_of_days) + return random_date + + def generate(self, user_id: int) -> list[Dict]: + """ + Because we are doing this work in parallel processes, we need a deterministic way to know what a purchase's ID should be given on the input of a user_id. + tldr; Every 10 user_ids produce 10 purchases. User ID x5 has no purchases, User ID mod x7 has 2, and everyone else has 1 + """ + + purchases: list[Dict] = [] + last_user_id_digit = int(repr(user_id)[-1]) + purchase_count = 1 + id_offset = 0 + if last_user_id_digit - 1 == 5: + purchase_count = 0 + elif last_user_id_digit - 1 == 6: + id_offset = 1 + elif last_user_id_digit - 1 == 7: + id_offset = 1 + purchase_count = 2 + + total_products = 100 + i = 0 + + while purchase_count > 0: + id = user_id + i + 1 - id_offset + time_a = dt.datetime() + time_b = dt.datetime() + created_at = time_a if time_a <= time_b else time_b + product_id = numeric.integer_number(1, total_products) + added_to_cart_at = self.random_date_in_range(created_at) + purchased_at = ( + self.random_date_in_range(added_to_cart_at) + if added_to_cart_at is not None and numeric.integer_number(1, 100) <= 70 + else None + ) # 70% likely to purchase the item in the cart + returned_at = ( + self.random_date_in_range(purchased_at) if purchased_at is not None and numeric.integer_number(1, 100) <= 15 else None + ) # 15% likely to return the item + + purchase = { + "id": id, + "product_id": product_id, + "user_id": user_id + 1, + "added_to_cart_at": format_airbyte_time(added_to_cart_at) if added_to_cart_at is not None else None, + "purchased_at": format_airbyte_time(purchased_at) if purchased_at is not None else None, + "returned_at": format_airbyte_time(returned_at) if returned_at is not None else None, + } + + record = AirbyteRecordMessage(stream=self.stream_name, data=purchase, emitted_at=now_millis()) + message = AirbyteMessageWithCachedJSON(type=Type.RECORD, record=record) + purchases.append(message) + + purchase_count = purchase_count - 1 + i += 1 + + return purchases diff --git a/airbyte-integrations/connectors/source-faker/source_faker/source.py b/airbyte-integrations/connectors/source-faker/source_faker/source.py index ef56bbfc192526..3772cea7551606 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/source.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/source.py @@ -19,14 +19,14 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> return False, "Count option is missing" def streams(self, config: Mapping[str, Any]) -> List[Stream]: - count: int = config["count"] if "count" in config else 0 seed: int = config["seed"] if "seed" in config else None records_per_sync: int = config["records_per_sync"] if "records_per_sync" in config else 500 records_per_slice: int = config["records_per_slice"] if "records_per_slice" in config else 100 + parallelism: int = config["parallelism"] if "parallelism" in config else 4 return [ - Products(count, seed, records_per_sync, records_per_slice), - Users(count, seed, records_per_sync, records_per_slice), - Purchases(seed, records_per_sync, records_per_slice), + Products(count, seed, parallelism, records_per_sync, records_per_slice), + Users(count, seed, parallelism, records_per_sync, records_per_slice), + Purchases(count, seed, parallelism, records_per_sync, records_per_slice), ] diff --git a/airbyte-integrations/connectors/source-faker/source_faker/spec.json b/airbyte-integrations/connectors/source-faker/source_faker/spec.json index 1018a957f1095e..0d20f791000da4 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/spec.json +++ b/airbyte-integrations/connectors/source-faker/source_faker/spec.json @@ -35,8 +35,16 @@ "description": "How many fake records will be in each page (stream slice), before a state message is emitted?", "type": "integer", "minimum": 1, - "default": 100, + "default": 1000, "order": 3 + }, + "parallelism": { + "title": "Parallelism", + "description": "How many parallel workers should we use to generate fake data? Choose a value equal to the number of CPUs you will allocate to this source.", + "type": "integer", + "minimum": 1, + "default": 4, + "order": 4 } } } diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index 352dfb2411db06..a25e9381361530 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -2,23 +2,22 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -import datetime import os +from multiprocessing import Pool from typing import Any, Dict, Iterable, Mapping, Optional -from airbyte_cdk.models import AirbyteEstimateTraceMessage, AirbyteTraceMessage, EstimateType, TraceType from airbyte_cdk.sources.streams import IncrementalMixin, Stream -from mimesis import Datetime, Numeric, Person -from mimesis.locales import Locale -from .utils import format_airbyte_time, read_json +from .purchase_generator import PurchaseGenerator +from .user_generator import UserGenerator +from .utils import generate_estimate, read_json class Products(Stream, IncrementalMixin): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, parallelism: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.seed = seed self.records_per_sync = records_per_sync @@ -64,14 +63,14 @@ class Users(Stream, IncrementalMixin): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, parallelism: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.count = count self.seed = seed self.records_per_sync = records_per_sync self.records_per_slice = records_per_slice - self.person = Person(locale=Locale.EN, seed=self.seed) - self.dt = Datetime(seed=self.seed) + self.parallelism = parallelism + self.generator = UserGenerator(self.name, self.seed) @property def state_checkpoint_interval(self) -> Optional[int]: @@ -88,74 +87,49 @@ def state(self) -> Mapping[str, Any]: def state(self, value: Mapping[str, Any]): self._state = value - def generate_user(self, user_id: int): - time_a = self.dt.datetime() - time_b = self.dt.datetime() - - profile = { - "id": user_id + 1, - "created_at": format_airbyte_time(time_a if time_a <= time_b else time_b), - "updated_at": format_airbyte_time(time_a if time_a > time_b else time_b), - "name": self.person.name(), - "title": self.person.title(), - "age": self.person.age(), - "email": self.person.email(), - "telephone": self.person.telephone(), - "gender": self.person.gender(), - "language": self.person.language(), - "academic_degree": self.person.academic_degree(), - "nationality": self.person.nationality(), - "occupation": self.person.occupation(), - "height": self.person.height(), - "blood_type": self.person.blood_type(), - "weight": self.person.weight(), - } - - while not profile["created_at"]: - profile["created_at"] = format_airbyte_time(self.dt.datetime()) - - if not profile["updated_at"]: - profile["updated_at"] = profile["created_at"] + 1 - - return profile - def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: + """ + This is a multi-process implementation of read_records. + We make N workers (where N is the number of available CPUs) and spread out the CPU-bound work of generating records and serializing them to JSON + """ + total_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 records_in_sync = 0 - records_in_slice = 0 median_record_byte_size = 450 yield generate_estimate(self.name, self.count - total_records, median_record_byte_size) - for i in range(total_records, self.count): - user = self.generate_user(i) - yield user - total_records += 1 - records_in_sync += 1 - records_in_slice += 1 + with Pool(initializer=self.generator.prepare, processes=self.parallelism) as pool: + while records_in_sync < self.count and records_in_sync < self.records_per_sync: + records_remaining_this_loop = min(self.records_per_slice, (self.count - total_records)) + if records_remaining_this_loop <= 0: + break + users = pool.map(self.generator.generate, range(total_records, total_records + records_remaining_this_loop)) + for user in users: + total_records += 1 + records_in_sync += 1 + yield user - if records_in_slice >= self.records_per_slice: - self.state = {self.cursor_field: total_records, "seed": self.seed} - records_in_slice = 0 + if records_in_sync >= self.records_per_sync: + break - if records_in_sync == self.records_per_sync: - break + self.state = {self.cursor_field: total_records, "seed": self.seed} self.state = {self.cursor_field: total_records, "seed": self.seed} - set_total_user_records(total_records) class Purchases(Stream, IncrementalMixin): primary_key = None - cursor_field = "user_id" + cursor_field = "id" - def __init__(self, seed: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, parallelism: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) + self.count = count self.seed = seed self.records_per_sync = records_per_sync self.records_per_slice = records_per_slice - self.dt = Datetime(seed=self.seed) - self.numeric = Numeric(seed=self.seed) + self.parallelism = parallelism + self.generator = PurchaseGenerator(self.name, self.seed) @property def state_checkpoint_interval(self) -> Optional[int]: @@ -172,86 +146,37 @@ def state(self) -> Mapping[str, Any]: def state(self, value: Mapping[str, Any]): self._state = value - def random_date_in_range( - self, start_date: datetime.datetime, end_date: datetime.datetime = datetime.datetime.now() - ) -> datetime.datetime: - time_between_dates = end_date - start_date - days_between_dates = time_between_dates.days - if days_between_dates < 2: - days_between_dates = 2 - random_number_of_days = self.numeric.integer_number(0, days_between_dates) - random_date = start_date + datetime.timedelta(days=random_number_of_days) - return random_date - - def generate_purchases(self, user_id: int, purchases_count: int) -> list[Dict]: - purchases: list[Dict] = [] - purchase_percent_remaining = 70 # ~30% of people will have no purchases - total_products = 100 - purchase_percent_remaining = purchase_percent_remaining - self.numeric.integer_number(1, 100) - i = 0 - - time_a = self.dt.datetime() - time_b = self.dt.datetime() - created_at = time_a if time_a <= time_b else time_b - - while purchase_percent_remaining > 0: - id = purchases_count + i + 1 - product_id = self.numeric.integer_number(1, total_products) - added_to_cart_at = self.random_date_in_range(created_at) - purchased_at = ( - self.random_date_in_range(added_to_cart_at) - if added_to_cart_at is not None and self.numeric.integer_number(1, 100) <= 70 - else None - ) # 70% likely to purchase the item in the cart - returned_at = ( - self.random_date_in_range(purchased_at) if purchased_at is not None and self.numeric.integer_number(1, 100) <= 15 else None - ) # 15% likely to return the item - - purchase = { - "id": id, - "product_id": product_id, - "user_id": user_id, - "added_to_cart_at": format_airbyte_time(added_to_cart_at) if added_to_cart_at is not None else None, - "purchased_at": format_airbyte_time(purchased_at) if purchased_at is not None else None, - "returned_at": format_airbyte_time(returned_at) if returned_at is not None else None, - } - purchases.append(purchase) - - purchase_percent_remaining = purchase_percent_remaining - self.numeric.integer_number(1, 100) - i += 1 - return purchases - def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: - purchases_count = self.state[self.cursor_field] if self.cursor_field in self.state else 0 + """ + This is a multi-process implementation of read_records. + We make N workers (where N is the number of available CPUs) and spread out the CPU-bound work of generating records and serializing them to JSON + """ - if total_user_records <= 0: - return # if there are no new users, there should be no new purchases + total_purchase_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 + total_user_records = self.state["user_id"] if "user_id" in self.state else 0 + user_records_in_sync = 0 + # a fuzzy guess, some users have purchases, some don't median_record_byte_size = 230 - yield generate_estimate( - self.name, total_user_records - purchases_count * 1.3, median_record_byte_size - ) # a fuzzy guess, some users have purchases, some don't - - for i in range(purchases_count, total_user_records): - purchases = self.generate_purchases(i + 1, purchases_count) - for purchase in purchases: - yield purchase - purchases_count += 1 - - self.state = {self.cursor_field: total_user_records, "seed": self.seed} - + yield generate_estimate(self.name, (self.count - total_user_records) * 1.3, median_record_byte_size) -def generate_estimate(stream_name: str, total: int, bytes_per_row: int): - emitted_at = int(datetime.datetime.now().timestamp() * 1000) - estimate_message = AirbyteEstimateTraceMessage( - type=EstimateType.STREAM, name=stream_name, row_estimate=round(total), byte_estimate=round(total * bytes_per_row) - ) - return AirbyteTraceMessage(type=TraceType.ESTIMATE, emitted_at=emitted_at, estimate=estimate_message) + with Pool(initializer=self.generator.prepare, processes=self.parallelism) as pool: + while total_user_records < self.count and user_records_in_sync < self.records_per_sync: + records_remaining_this_loop = min(self.records_per_slice, (self.count - user_records_in_sync)) + if records_remaining_this_loop <= 0: + break + carts = pool.map(self.generator.generate, range(total_user_records, total_user_records + records_remaining_this_loop)) + for purchases in carts: + for purchase in purchases: + total_purchase_records += 1 + yield purchase + total_user_records += 1 + user_records_in_sync += 1 -# a globals hack to share data between streams: -total_user_records = 0 + if user_records_in_sync >= self.records_per_sync: + break + self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} -def set_total_user_records(total: int): - globals()["total_user_records"] = total + self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} diff --git a/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py new file mode 100644 index 00000000000000..3fcedd2b6ad41e --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py @@ -0,0 +1,72 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from multiprocessing import current_process + +from airbyte_cdk.models import AirbyteRecordMessage, Type +from mimesis import Datetime, Person +from mimesis.locales import Locale + +from .airbyte_message_with_cached_json import AirbyteMessageWithCachedJSON +from .utils import format_airbyte_time, now_millis + + +class UserGenerator: + def __init__(self, stream_name: str, seed: int) -> None: + self.stream_name = stream_name + self.seed = seed + + def prepare(self): + """ + Note: the instances of the mimesis generators need to be global. + Yes, they *should* be able to be instance variables on this class, which should only instantiated once-per-worker, but that's not quite the case: + * relying only on prepare as a pool initializer fails because we are calling the parent process's method, not the fork + * Calling prepare() as part of generate() (perhaps checking if self.person is set) and then `print(self, current_process()._identity, current_process().pid)` reveals multiple object IDs in the same process, resetting the internal random counters + """ + + seed_with_offset = self.seed + if self.seed is not None and len(current_process()._identity) > 0: + seed_with_offset = self.seed + current_process()._identity[0] + + global person + global dt + + person = Person(locale=Locale.EN, seed=seed_with_offset) + dt = Datetime(seed=seed_with_offset) + + def generate(self, user_id: int): + time_a = dt.datetime() + time_b = dt.datetime() + + # faker doesn't always produce unique email addresses, so to enforce uniqueness, we will append the user_id to the prefix + email_parts = person.email().split("@") + email = f"{email_parts[0]}+{user_id + 1}@{email_parts[1]}" + + profile = { + "id": user_id + 1, + "created_at": format_airbyte_time(time_a if time_a <= time_b else time_b), + "updated_at": format_airbyte_time(time_a if time_a > time_b else time_b), + "name": person.name(), + "title": person.title(), + "age": person.age(), + "email": email, + "telephone": person.telephone(), + "gender": person.gender(), + "language": person.language(), + "academic_degree": person.academic_degree(), + "nationality": person.nationality(), + "occupation": person.occupation(), + "height": person.height(), + "blood_type": person.blood_type(), + "weight": person.weight(), + } + + while not profile["created_at"]: + profile["created_at"] = format_airbyte_time(dt.datetime()) + + if not profile["updated_at"]: + profile["updated_at"] = profile["created_at"] + 1 + + record = AirbyteRecordMessage(stream=self.stream_name, data=profile, emitted_at=now_millis()) + return AirbyteMessageWithCachedJSON(type=Type.RECORD, record=record) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/utils.py b/airbyte-integrations/connectors/source-faker/source_faker/utils.py index 12970853c9569f..9c094b11bcb7ee 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/utils.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/utils.py @@ -5,6 +5,8 @@ import datetime import json +from airbyte_cdk.models import AirbyteEstimateTraceMessage, AirbyteTraceMessage, EstimateType, TraceType + def read_json(filepath): with open(filepath, "r") as f: @@ -17,3 +19,15 @@ def format_airbyte_time(d: datetime): s = s.replace(" ", "T") s += "+00:00" return s + + +def now_millis(): + return int(datetime.datetime.now().timestamp() * 1000) + + +def generate_estimate(stream_name: str, total: int, bytes_per_row: int): + emitted_at = int(datetime.datetime.now().timestamp() * 1000) + estimate_message = AirbyteEstimateTraceMessage( + type=EstimateType.STREAM, name=stream_name, row_estimate=round(total), byte_estimate=round(total * bytes_per_row) + ) + return AirbyteTraceMessage(type=TraceType.ESTIMATE, emitted_at=emitted_at, estimate=estimate_message) diff --git a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py index 8a989b7778ac6e..193960b734531f 100644 --- a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py @@ -25,7 +25,7 @@ def exception(a,b,**kwargs): def schemas_are_valid(): source = SourceFaker() - config = {"count": 1} + config = {"count": 1, "parallelism": 1} catalog = source.discover(None, config) catalog = AirbyteMessage(type=Type.CATALOG, catalog=catalog).dict(exclude_unset=True) schemas = [stream["json_schema"] for stream in catalog["catalog"]["streams"]] @@ -36,7 +36,7 @@ def schemas_are_valid(): def test_source_streams(): source = SourceFaker() - config = {"count": 1} + config = {"count": 1, "parallelism": 1} catalog = source.discover(None, config) catalog = AirbyteMessage(type=Type.CATALOG, catalog=catalog).dict(exclude_unset=True) schemas = [stream["json_schema"] for stream in catalog["catalog"]["streams"]] @@ -64,7 +64,7 @@ def test_source_streams(): def test_read_small_random_data(): source = SourceFaker() - config = {"count": 10} + config = {"count": 10, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -98,7 +98,7 @@ def test_read_small_random_data(): def test_no_read_limit_hit(): source = SourceFaker() - config = {"count": 10} + config = {"count": 10, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -128,7 +128,7 @@ def test_no_read_limit_hit(): def test_read_big_random_data(): source = SourceFaker() - config = {"count": 1000, "records_per_slice": 100, "records_per_sync": 1000} + config = {"count": 1000, "records_per_slice": 100, "records_per_sync": 1000, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -163,7 +163,7 @@ def test_read_big_random_data(): def test_with_purchases(): source = SourceFaker() - config = {"count": 1000, "records_per_sync": 1000} + config = {"count": 1000, "records_per_sync": 1000, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -205,7 +205,7 @@ def test_with_purchases(): def test_sync_ends_with_limit(): source = SourceFaker() - config = {"count": 100, "records_per_sync": 5} + config = {"count": 100, "records_per_sync": 5, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -239,7 +239,7 @@ def test_read_with_seed(): """ source = SourceFaker() - config = {"count": 1, "seed": 100} + config = {"count": 1, "seed": 100, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -253,14 +253,14 @@ def test_read_with_seed(): iterator = source.read(logger, config, catalog, state) records = [row for row in iterator if row.type is Type.RECORD] - assert records[0].record.data["occupation"] == "Roadworker" - assert records[0].record.data["email"] == "reproduce1856@outlook.com" + assert records[0].record.data["occupation"] == "Cartoonist" + assert records[0].record.data["email"] == "reflect1958+1@yahoo.com" def test_ensure_no_purchases_without_users(): with pytest.raises(ValueError): source = SourceFaker() - config = {"count": 100} + config = {"count": 100, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ {"stream": {"name": "purchases", "json_schema": {}}, "sync_mode": "incremental", "destination_sync_mode": "overwrite"}, diff --git a/docs/integrations/sources/faker.md b/docs/integrations/sources/faker.md index 4ea89e33da2344..ef69a14377cb80 100644 --- a/docs/integrations/sources/faker.md +++ b/docs/integrations/sources/faker.md @@ -81,17 +81,18 @@ None! ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- | -| 1.0.0 | 2022-11-28 | [19490](https://github.com/airbytehq/airbyte/pull/19490) | Faker uses the CDK; rename streams to be lower-case (breaking), add determinism to random purchases, and rename | -| 0.2.1 | 2022-10-14 | [19197](https://github.com/airbytehq/airbyte/pull/19197) | Emit `AirbyteEstimateTraceMessage` | -| 0.2.0 | 2022-10-14 | [18021](https://github.com/airbytehq/airbyte/pull/18021) | Move to mimesis for speed! | -| 0.1.8 | 2022-10-12 | [17889](https://github.com/airbytehq/airbyte/pull/17889) | Bump to test publish command (2) | -| 0.1.7 | 2022-10-11 | [17848](https://github.com/airbytehq/airbyte/pull/17848) | Bump to test publish command | -| 0.1.6 | 2022-09-07 | [16418](https://github.com/airbytehq/airbyte/pull/16418) | Log start of each stream | -| 0.1.5 | 2022-06-10 | [13695](https://github.com/airbytehq/airbyte/pull/13695) | Emit timestamps in the proper ISO format | -| 0.1.4 | 2022-05-27 | [13298](https://github.com/airbytehq/airbyte/pull/13298) | Test publication flow | -| 0.1.3 | 2022-05-27 | [13248](https://github.com/airbytehq/airbyte/pull/13248) | Add options for records_per_sync and page_size | -| 0.1.2 | 2022-05-26 | [13248](https://github.com/airbytehq/airbyte/pull/13293) | Test publication flow | -| 0.1.1 | 2022-05-26 | [13235](https://github.com/airbytehq/airbyte/pull/13235) | Publish for AMD and ARM (M1 Macs) & remove User.birthdate | -| 0.1.0 | 2022-04-12 | [11738](https://github.com/airbytehq/airbyte/pull/11738) | The Faker Source is created | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- | +| 2.0.0 | 2022-12-14 | [20492](https://github.com/airbytehq/airbyte/pull/20492) and [20741](https://github.com/airbytehq/airbyte/pull/20741) | Decouple stream states for better parallelism | +| 1.0.0 | 2022-11-28 | [19490](https://github.com/airbytehq/airbyte/pull/19490) | Faker uses the CDK; rename streams to be lower-case (breaking), add determinism to random purchases, and rename | +| 0.2.1 | 2022-10-14 | [19197](https://github.com/airbytehq/airbyte/pull/19197) | Emit `AirbyteEstimateTraceMessage` | +| 0.2.0 | 2022-10-14 | [18021](https://github.com/airbytehq/airbyte/pull/18021) | Move to mimesis for speed! | +| 0.1.8 | 2022-10-12 | [17889](https://github.com/airbytehq/airbyte/pull/17889) | Bump to test publish command (2) | +| 0.1.7 | 2022-10-11 | [17848](https://github.com/airbytehq/airbyte/pull/17848) | Bump to test publish command | +| 0.1.6 | 2022-09-07 | [16418](https://github.com/airbytehq/airbyte/pull/16418) | Log start of each stream | +| 0.1.5 | 2022-06-10 | [13695](https://github.com/airbytehq/airbyte/pull/13695) | Emit timestamps in the proper ISO format | +| 0.1.4 | 2022-05-27 | [13298](https://github.com/airbytehq/airbyte/pull/13298) | Test publication flow | +| 0.1.3 | 2022-05-27 | [13248](https://github.com/airbytehq/airbyte/pull/13248) | Add options for records_per_sync and page_size | +| 0.1.2 | 2022-05-26 | [13248](https://github.com/airbytehq/airbyte/pull/13293) | Test publication flow | +| 0.1.1 | 2022-05-26 | [13235](https://github.com/airbytehq/airbyte/pull/13235) | Publish for AMD and ARM (M1 Macs) & remove User.birthdate | +| 0.1.0 | 2022-04-12 | [11738](https://github.com/airbytehq/airbyte/pull/11738) | The Faker Source is created |