-
Notifications
You must be signed in to change notification settings - Fork 3.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
馃毃Source Smartsheets - add support for ingesting metadata fields (#22382)
* Adds metadata field * update requirements to support for python 3.10 * update sample config to reflect metadata fields * modify spec and sheet to include metadata fields * update tests to account for metadata fields * update using python 3.10 base image * fix linting errors * update test files for integrations tests * update readme to include info on metadata fields * update acceptance-test yaml format * update docs include metadata info and change log * Adds metadata field * update requirements to support for python 3.10 * update sample config to reflect metadata fields * modify spec and sheet to include metadata fields * update tests to account for metadata fields * update using python 3.10 base image * fix linting errors * update test files for integrations tests * update readme to include info on metadata fields * resolve merge conflicts * update docs include metadata info and change log * address merge conflict issues * update sample config files for integrations tests * fix spec and acceptance tests config * revert image change from python 3.10 to 3.9 * update metadata fields in spec to enum * bump version --------- Co-authored-by: jeanluciano <jeanluciano@mailbox.org> Co-authored-by: Sajarin <sajarindider@gmail.com> Co-authored-by: Mal Hancock <mallory@archangelic.space>
- Loading branch information
4 people
committed
Jun 2, 2023
1 parent
096d9e8
commit 2995ced
Showing
17 changed files
with
340 additions
and
37 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
airbyte-integrations/connectors/source-smartsheets/integration_tests/abnormal_state.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| { | ||
| "aws_s3_sample": { | ||
| "modifiedAt": "2222-03-07T11:30:00+00:00" | ||
| "modifiedAt": "2250-03-07T11:30:00+00:00" | ||
| } | ||
| } |
49 changes: 25 additions & 24 deletions
49
airbyte-integrations/connectors/source-smartsheets/integration_tests/configured_catalog.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,27 +1,28 @@ | ||
| { | ||
| "streams": [ | ||
| { | ||
| "stream": { | ||
| "name": "aws_s3_sample", | ||
| "json_schema": { | ||
| "$schema": "http://json-schema.org/draft-07/schema#", | ||
| "type": "object", | ||
| "properties": { | ||
| "id": { "type": "string" }, | ||
| "first_name": { "type": "string" }, | ||
| "last_name": { "type": "string" }, | ||
| "gender": { "type": "string" }, | ||
| "ip_address": { "type": "string" }, | ||
| "primary_email": { "type": "string" }, | ||
| "dob": { "type": "string", "format": "date" }, | ||
| "modifiedAt": { "type": "string", "format": "date-time" } | ||
| } | ||
| "streams": [ | ||
| { | ||
| "stream": { | ||
| "name": "aws_s3_sample", | ||
| "json_schema": { | ||
| "$schema": "http://json-schema.org/draft-07/schema#", | ||
| "type": "object", | ||
| "properties": { | ||
| "id": { "type": "string" }, | ||
| "first_name": { "type": "string" }, | ||
| "last_name": { "type": "string" }, | ||
| "gender": { "type": "string" }, | ||
| "ip_address": { "type": "string" }, | ||
| "primary_email": { "type": "string" }, | ||
| "dob": { "type": "string", "format": "date" }, | ||
| "modifiedAt": { "type": "string", "format": "date-time" } | ||
| } | ||
| }, | ||
| "supported_sync_modes": ["full_refresh"] | ||
| }, | ||
| "supported_sync_modes": ["full_refresh"] | ||
| }, | ||
| "sync_mode": "full_refresh", | ||
| "cursor_field": null, | ||
| "destination_sync_mode": "overwrite" | ||
| } | ||
| ] | ||
| "sync_mode": "full_refresh", | ||
| "cursor_field": null, | ||
| "destination_sync_mode": "overwrite" | ||
| } | ||
| ] | ||
| } | ||
|
|
3 changes: 2 additions & 1 deletion
3
airbyte-integrations/connectors/source-smartsheets/integration_tests/invalid_config.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,5 @@ | ||
| { | ||
| "access_token": "fake-token", | ||
| "spreadsheet_id": "fake-id" | ||
| "spreadsheet_id": "fake-id", | ||
| "metadata_fields": ["fake", 33] | ||
| } |
21 changes: 20 additions & 1 deletion
21
airbyte-integrations/connectors/source-smartsheets/integration_tests/sample_config.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,23 @@ | ||
| { | ||
| "access_token": "access-token", | ||
| "spreadsheet_id": "spreadsheet-id" | ||
| "spreadsheet_id": "spreadsheet-id", | ||
| "metadata_fields": [ | ||
| "sheetcreatedAt", | ||
| "sheetid", | ||
| "sheetmodifiedAt", | ||
| "sheetname", | ||
| "sheetpermalink", | ||
| "sheetversion", | ||
| "sheetaccess_level", | ||
| "row_id", | ||
| "row_access_level", | ||
| "row_created_at", | ||
| "row_created_by", | ||
| "row_expanded", | ||
| "row_modified_by", | ||
| "row_parent_id", | ||
| "row_permalink", | ||
| "row_number", | ||
| "row_version" | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
5 changes: 5 additions & 0 deletions
5
airbyte-integrations/connectors/source-smartsheets/sample_files/abnormal_state.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| { | ||
| "Airbyte-connector Test": { | ||
| "modifiedAt": "2250-03-07T11:30:00+00:00" | ||
| } | ||
| } |
61 changes: 61 additions & 0 deletions
61
airbyte-integrations/connectors/source-smartsheets/sample_files/configured_catalog.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| { | ||
| "streams": [ | ||
| { | ||
| "sync_mode": "full_refresh", | ||
| "destination_sync_mode": "overwrite", | ||
| "stream": { | ||
| "name": "Airbyte-connector Test", | ||
| "supported_sync_modes": [ | ||
| "full_refresh", | ||
| "incremental" | ||
| ], | ||
| "source_defined_cursor": false, | ||
| "source_defined_primary_key": [["race"]], | ||
| "json_schema": { | ||
| "$schema": "http://json-schema.org/draft-07/schema#", | ||
| "type": "object", | ||
| "properties": { | ||
| "race": { | ||
| "type": "string" | ||
| }, | ||
| "height": { | ||
| "type": "string" | ||
| }, | ||
| "weight": { | ||
| "type": "string" | ||
| }, | ||
| "speed": { | ||
| "type": "string" | ||
| }, | ||
| "strength": { | ||
| "type": "string" | ||
| }, | ||
| "dexterity": { | ||
| "type": "string" | ||
| }, | ||
| "constitution": { | ||
| "type": "string" | ||
| }, | ||
| "intelligence": { | ||
| "type": "string" | ||
| }, | ||
| "wisdom": { | ||
| "type": "string" | ||
| }, | ||
| "charisma": { | ||
| "type": "string" | ||
| }, | ||
| "modifiedAt": { | ||
| "type": "string", | ||
| "format": "date-time" | ||
| }, | ||
| "row_id": { | ||
| "type": "string" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| ] | ||
| } | ||
|
|
Oops, something went wrong.