-
Notifications
You must be signed in to change notification settings - Fork 4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
🚨🚨 Source SFTP Bulk: migrate to file-based CDK (#36256)
Signed-off-by: Artem Inzhyyants <artem.inzhyyants@gmail.com>
- Loading branch information
Showing
39 changed files
with
3,580 additions
and
1,006 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
[run] | ||
omit = | ||
source_sftp_bulk/run.py |
17 changes: 0 additions & 17 deletions
17
airbyte-integrations/connectors/source-sftp-bulk/Dockerfile
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
31 changes: 12 additions & 19 deletions
31
airbyte-integrations/connectors/source-sftp-bulk/acceptance-test-config.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,27 +1,20 @@ | ||
# See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) | ||
# for more information about how to configure these tests | ||
connector_image: airbyte/source-sftp-bulk:dev | ||
tests: | ||
acceptance_tests: | ||
spec: | ||
- spec_path: "source_sftp_bulk/spec.json" | ||
timeout_seconds: 60 | ||
tests: | ||
- spec_path: "integration_tests/spec.json" | ||
timeout_seconds: 60 | ||
backward_compatibility_tests_config: | ||
disable_for_version: 0.3.2 # `start_date` format changed to format: date-time | ||
connection: | ||
- config_path: "integration_tests/valid_config.json" | ||
status: "succeed" | ||
timeout_seconds: 60 | ||
- config_path: "integration_tests/invalid_config.json" | ||
status: "failed" | ||
timeout_seconds: 60 | ||
bypass_reason: "This connector uses integration tests" | ||
discovery: | ||
- config_path: "integration_tests/valid_config.json" | ||
bypass_reason: "This connector uses integration tests" | ||
basic_read: | ||
- config_path: "integration_tests/valid_config.json" | ||
configured_catalog_path: "integration_tests/configured_catalog.json" | ||
empty_streams: [] | ||
incremental: | ||
- config_path: "integration_tests/valid_config.json" | ||
configured_catalog_path: "integration_tests/configured_catalog.json" | ||
future_state_path: "integration_tests/abnormal_state.json" | ||
bypass_reason: "This connector uses integration tests" | ||
full_refresh: | ||
- config_path: "integration_tests/valid_config.json" | ||
configured_catalog_path: "integration_tests/configured_catalog.json" | ||
bypass_reason: "This connector uses integration tests" | ||
incremental: | ||
bypass_reason: "This connector uses integration tests" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
54 changes: 54 additions & 0 deletions
54
...e-integrations/connectors/source-sftp-bulk/integration_tests/configs/config_password.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
{ | ||
"host": "localhost", | ||
"port": 2222, | ||
"username": "foo", | ||
"credentials": { | ||
"auth_type": "password", | ||
"password": "pass" | ||
}, | ||
"file_type": "json", | ||
"start_date": "2021-01-01T00:00:00.000000Z", | ||
"folder_path": "/files", | ||
"streams": [ | ||
{ | ||
"name": "test_stream", | ||
"file_type": "csv", | ||
"globs": ["**/test_1.csv"], | ||
"legacy_prefix": "", | ||
"validation_policy": "Emit Record", | ||
"format": { | ||
"filetype": "csv", | ||
"delimiter": ",", | ||
"quote_char": "\"", | ||
"double_quote": true, | ||
"null_values": [ | ||
"", | ||
"#N/A", | ||
"#N/A N/A", | ||
"#NA", | ||
"-1.#IND", | ||
"-1.#QNAN", | ||
"-NaN", | ||
"-nan", | ||
"1.#IND", | ||
"1.#QNAN", | ||
"N/A", | ||
"NA", | ||
"NULL", | ||
"NaN", | ||
"n/a", | ||
"nan", | ||
"null" | ||
], | ||
"true_values": ["1", "True", "TRUE", "true"], | ||
"false_values": ["0", "False", "FALSE", "false"], | ||
"inference_type": "Primitive Types Only", | ||
"strings_can_be_null": false, | ||
"encoding": "utf8", | ||
"header_definition": { | ||
"header_definition_type": "From CSV" | ||
} | ||
} | ||
} | ||
] | ||
} |
54 changes: 54 additions & 0 deletions
54
...ntegrations/connectors/source-sftp-bulk/integration_tests/configs/config_private_key.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
{ | ||
"host": "localhost", | ||
"port": 2222, | ||
"username": "foo", | ||
"credentials": { | ||
"auth_type": "private_key", | ||
"private_key": "key" | ||
}, | ||
"file_type": "json", | ||
"start_date": "2021-01-01T00:00:00.000000Z", | ||
"folder_path": "/files", | ||
"streams": [ | ||
{ | ||
"name": "test_stream", | ||
"file_type": "csv", | ||
"globs": ["**/test_1.csv"], | ||
"legacy_prefix": "", | ||
"validation_policy": "Emit Record", | ||
"format": { | ||
"filetype": "csv", | ||
"delimiter": ",", | ||
"quote_char": "\"", | ||
"double_quote": true, | ||
"null_values": [ | ||
"", | ||
"#N/A", | ||
"#N/A N/A", | ||
"#NA", | ||
"-1.#IND", | ||
"-1.#QNAN", | ||
"-NaN", | ||
"-nan", | ||
"1.#IND", | ||
"1.#QNAN", | ||
"N/A", | ||
"NA", | ||
"NULL", | ||
"NaN", | ||
"n/a", | ||
"nan", | ||
"null" | ||
], | ||
"true_values": ["1", "True", "TRUE", "true"], | ||
"false_values": ["0", "False", "FALSE", "false"], | ||
"inference_type": "Primitive Types Only", | ||
"strings_can_be_null": false, | ||
"encoding": "utf8", | ||
"header_definition": { | ||
"header_definition_type": "From CSV" | ||
} | ||
} | ||
} | ||
] | ||
} |
44 changes: 44 additions & 0 deletions
44
airbyte-integrations/connectors/source-sftp-bulk/integration_tests/configs/stream_csv.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
{ | ||
"streams": [ | ||
{ | ||
"name": "test_stream", | ||
"file_type": "csv", | ||
"globs": ["**/test_1.csv", "**/test_3.csv"], | ||
"legacy_prefix": "", | ||
"validation_policy": "Emit Record", | ||
"format": { | ||
"filetype": "csv", | ||
"delimiter": ",", | ||
"quote_char": "\"", | ||
"double_quote": true, | ||
"null_values": [ | ||
"", | ||
"#N/A", | ||
"#N/A N/A", | ||
"#NA", | ||
"-1.#IND", | ||
"-1.#QNAN", | ||
"-NaN", | ||
"-nan", | ||
"1.#IND", | ||
"1.#QNAN", | ||
"N/A", | ||
"NA", | ||
"NULL", | ||
"NaN", | ||
"n/a", | ||
"nan", | ||
"null" | ||
], | ||
"true_values": ["1", "True", "TRUE", "true"], | ||
"false_values": ["0", "False", "FALSE", "false"], | ||
"inference_type": "Primitive Types Only", | ||
"strings_can_be_null": false, | ||
"encoding": "utf8", | ||
"header_definition": { | ||
"header_definition_type": "From CSV" | ||
} | ||
} | ||
} | ||
] | ||
} |
Oops, something went wrong.