From 1d5b916d2f0ea1063d35709a58ff9db5ce0c842f Mon Sep 17 00:00:00 2001 From: "Sherif A. Nada" Date: Wed, 27 Jan 2021 14:43:29 -0800 Subject: [PATCH 1/8] Make FB Rate limit problems clearer in FB connector docs (#1854) Co-authored-by: Shrif Nada Co-authored-by: Jared Rhizor --- docs/integrations/sources/appstore.md | 19 ++++++++++--------- docs/integrations/sources/drift.md | 2 +- .../sources/facebook-marketing.md | 9 +++++---- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/docs/integrations/sources/appstore.md b/docs/integrations/sources/appstore.md index 909633366dc3e..b2bebe966e073 100644 --- a/docs/integrations/sources/appstore.md +++ b/docs/integrations/sources/appstore.md @@ -2,14 +2,14 @@ ## Sync overview -This source can sync data for the [Appstore API](https://developer.apple.com/documentation/appstoreconnectapi). It supports only Incremental syncs. -The Applestore API is available for [many types of services](https://developer.apple.com/documentation/appstoreconnectapi), however this integration focuses only on the 'Reporting' service, from where data is extracted from. +This source can sync data for the [Appstore API](https://developer.apple.com/documentation/appstoreconnectapi). It supports only Incremental syncs. The Applestore API is available for [many types of services](https://developer.apple.com/documentation/appstoreconnectapi), however this integration focuses only on the 'Reporting' service, from where data is extracted from. Under the 'Reporting' Service, Appstore API has four different categories of Endpoints available. -1. Sales and Trends => [API docs](https://developer.apple.com/documentation/appstoreconnectapi/download_sales_and_trends_reports); ["UI docs"](https://help.apple.com/app-store-connect/#/dev061699fdb) -2. Finance Reports => [API docs](https://developer.apple.com/documentation/appstoreconnectapi/download_finance_reports); ["UI docs"](https://help.apple.com/app-store-connect/#/dev716cf3a0d) -3. Get Power and Performance Metrics for an App => [API docs](https://developer.apple.com/documentation/appstoreconnectapi/get_power_and_performance_metrics_for_an_app); -4. Get Power and Performance Metrics for a Build => [API docs](https://developer.apple.com/documentation/appstoreconnectapi/get_power_and_performance_metrics_for_a_build); + +1. Sales and Trends => [API docs](https://developer.apple.com/documentation/appstoreconnectapi/download_sales_and_trends_reports); ["UI docs"](https://help.apple.com/app-store-connect/#/dev061699fdb) +2. Finance Reports => [API docs](https://developer.apple.com/documentation/appstoreconnectapi/download_finance_reports); ["UI docs"](https://help.apple.com/app-store-connect/#/dev716cf3a0d) +3. Get Power and Performance Metrics for an App => [API docs](https://developer.apple.com/documentation/appstoreconnectapi/get_power_and_performance_metrics_for_an_app); +4. Get Power and Performance Metrics for a Build => [API docs](https://developer.apple.com/documentation/appstoreconnectapi/get_power_and_performance_metrics_for_a_build); This Source Connector is based on a [Singer Tap](https://github.com/miroapp/tap-appstore). @@ -19,7 +19,7 @@ This Source is capable of syncing the following "Sales and Trends" Streams: * [SALES](https://help.apple.com/app-store-connect/#/dev15f9508ca) * [SUBSCRIPTION](https://help.apple.com/app-store-connect/#/itc5dcdf6693) -* [SUBSCRIPTION_EVENT](https://help.apple.com/app-store-connect/#/itc0b9b9d5b2) +* [SUBSCRIPTION\_EVENT](https://help.apple.com/app-store-connect/#/itc0b9b9d5b2) * [SUBSCRIBER](https://help.apple.com/app-store-connect/#/itcf20f3392e) ### Data type mapping @@ -53,11 +53,12 @@ One issue that can happen is the API not having the data available for the perio ### Requirements * Key ID -* Key File (Private API Key) +* Key File \(Private API Key\) * Issuer ID * Vendor ID -* Start Date (The date that will be used in the first sync. Apple only allows to go back 365 days from today.) +* Start Date \(The date that will be used in the first sync. Apple only allows to go back 365 days from today.\) ### Setup guide Generate/Find all requirements using this [external article](https://leapfin.com/blog/apple-appstore-integration/). + diff --git a/docs/integrations/sources/drift.md b/docs/integrations/sources/drift.md index 384b05dda15a2..f56d170c87796 100644 --- a/docs/integrations/sources/drift.md +++ b/docs/integrations/sources/drift.md @@ -38,5 +38,5 @@ The Drift connector should not run into Drift API limitations under normal usage ### Setup guide -Follow Drift's [Setting Things Up ](https://devdocs.drift.com/docs/quick-start)guide for a more detailed description of how to obtain the API token. +Follow Drift's [Setting Things Up ](https://devdocs.drift.com/docs/quick-start)guide for a more detailed description of how to obtain the API token. diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index 2d48939eab886..a40f80c4b1b9f 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -43,11 +43,13 @@ For more information, see the [Facebook Insights API documentation. ](https://de | Full Refresh Sync | Yes | | | Incremental Sync | Yes | except AdCreatives | -### Performance considerations +### Rate Limiting & Performance Considerations -**Important note:** In order for data synced from your Facebook account to be up to date, you might need to apply with Facebook to upgrade your access token to the Ads Management Standard Tier as specified in the [Facebook Access documentation](https://developers.facebook.com/docs/marketing-api/access). Otherwise, Facebook might throttle Airbyte syncs, since the default tier \(Dev Access\) is heavily throttled by Facebook. +{% hint style="info" %} +Facebook heavily throttles API tokens generated from "Dev" Facebook Apps, making it infeasible to use such a token for syncs with Airbyte. To be able to use this connector, make sure to use an App with at least the "Business Standard" status to generate the API token below. +{% endhint %} -Note that Airbyte can adapt to throttling from Facebook. In the worst case scenario syncs from Facebook will take longer to complete and data will be less fresh. +See Facebook's [documentation on rate limiting](https://developers.facebook.com/docs/marketing-apis/rate-limiting) for more information. ## Getting started @@ -82,4 +84,3 @@ In the App Dashboard screen, click Marketing API --> Tools on the left sideba ![](../../.gitbook/assets/facebook_access_token.png) With the Ad Account ID and API access token, you should be ready to start pulling data from the Facebook Marketing API. Head to the Airbyte UI to setup your source connector! - From 143f880c785fbdec120dec92bb793132e12c22c8 Mon Sep 17 00:00:00 2001 From: vitaliizazmic <75620293+vitaliizazmic@users.noreply.github.com> Date: Thu, 28 Jan 2021 02:44:11 +0200 Subject: [PATCH 2/8] Tempo source #1764 - enable standard tests (#1829) --- .github/workflows/test-command.yml | 1 + airbyte-integrations/connectors/source-tempo/build.gradle | 4 ++-- tools/bin/ci_credentials.sh | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index a6514ecc5d688..ad44e749aab36 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -73,6 +73,7 @@ jobs: SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG: ${{ secrets.SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG }} SOURCE_RECURLY_INTEGRATION_TEST_CREDS: ${{ secrets.SOURCE_RECURLY_INTEGRATION_TEST_CREDS }} STRIPE_INTEGRATION_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_TEST_CREDS }} + TEMPO_INTEGRATION_TEST_CREDS: ${{ secrets.TEMPO_INTEGRATION_TEST_CREDS }} TWILIO_TEST_CREDS: ${{ secrets.TWILIO_TEST_CREDS }} ZENDESK_SECRETS_CREDS: ${{ secrets.ZENDESK_SECRETS_CREDS }} ZOOM_INTEGRATION_TEST_CREDS: ${{ secrets.ZOOM_INTEGRATION_TEST_CREDS }} diff --git a/airbyte-integrations/connectors/source-tempo/build.gradle b/airbyte-integrations/connectors/source-tempo/build.gradle index 6fa0b91d40f97..af7d80e967927 100644 --- a/airbyte-integrations/connectors/source-tempo/build.gradle +++ b/airbyte-integrations/connectors/source-tempo/build.gradle @@ -1,8 +1,8 @@ plugins { id 'airbyte-python' id 'airbyte-docker' -// id 'airbyte-source-test' -// id 'airbyte-integration-test-java' + id 'airbyte-source-test' + id 'airbyte-integration-test-java' } airbytePython { diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index c6e8cc6ddf8f9..5b856e027c99a 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -50,6 +50,7 @@ write_standard_creds source-sendgrid "$SENDGRID_INTEGRATION_TEST_CREDS" write_standard_creds source-shopify-singer "$SHOPIFY_INTEGRATION_TEST_CREDS" write_standard_creds source-slack-singer "$SLACK_TEST_CREDS" write_standard_creds source-stripe-singer "$STRIPE_INTEGRATION_TEST_CREDS" +write_standard_creds source-tempo "$TEMPO_INTEGRATION_TEST_CREDS" write_standard_creds source-twilio-singer "$TWILIO_TEST_CREDS" write_standard_creds source-zendesk-support-singer "$ZENDESK_SECRETS_CREDS" write_standard_creds source-zoom-singer "$ZOOM_INTEGRATION_TEST_CREDS" From a54dd8941849976409c66bd5ef1e61e5ae9427f4 Mon Sep 17 00:00:00 2001 From: Eugene K <34233075+eugene-kulak@users.noreply.github.com> Date: Wed, 27 Jan 2021 21:49:33 -0300 Subject: [PATCH 3/8] Adopt connector best practices for File Source (providers) #1584 (#1738) Co-authored-by: Sherif Nada --- .github/workflows/test-command.yml | 1 + .../778daa7c-feaf-4db6-96f3-70fd645acc77.json | 2 +- .../resources/seed/source_definitions.yaml | 2 +- .../connectors/source-file/Dockerfile | 2 +- .../connectors/source-file/Dockerfile.test | 2 +- .../connectors/source-file/build.gradle | 9 + .../source-file/integration_tests/__init__.py | 3 +- .../client_storage_providers_test.py | 145 +++++++ .../source-file/integration_tests/config.json | 2 +- .../{catalog.json => configured_catalog.json} | 0 .../source-file/integration_tests/conftest.py | 184 +++++++++ .../integration_tests/docker-compose.yml | 9 + .../integration_source_test.py | 195 --------- .../integration_tests/nested_config.json | 2 +- .../integration_tests/sample_files/empty.csv | 0 .../integration_tests/sample_files/test.csv | 3 + .../sample_files/test.csv.gz | Bin 0 -> 60 bytes .../integration_tests/sample_files/test.pkl | Bin 0 -> 1033 bytes .../sample_files/test.pkl.gz | Bin 0 -> 566 bytes .../integration_tests/standard_source_test.py | 25 +- .../connectors/source-file/setup.py | 33 +- .../source-file/source_file/client.py | 341 +++++++++++++++ .../source-file/source_file/source.py | 389 ++---------------- .../source-file/source_file/spec.json | 49 ++- .../source-file/unit_tests/unit_test.py | 4 +- docs/integrations/sources/file.md | 23 +- tools/bin/ci_credentials.sh | 2 +- 27 files changed, 798 insertions(+), 629 deletions(-) create mode 100644 airbyte-integrations/connectors/source-file/integration_tests/client_storage_providers_test.py rename airbyte-integrations/connectors/source-file/integration_tests/{catalog.json => configured_catalog.json} (100%) create mode 100644 airbyte-integrations/connectors/source-file/integration_tests/conftest.py create mode 100644 airbyte-integrations/connectors/source-file/integration_tests/docker-compose.yml delete mode 100644 airbyte-integrations/connectors/source-file/integration_tests/integration_source_test.py create mode 100644 airbyte-integrations/connectors/source-file/integration_tests/sample_files/empty.csv create mode 100644 airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.csv create mode 100644 airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.csv.gz create mode 100644 airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.pkl create mode 100644 airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.pkl.gz create mode 100644 airbyte-integrations/connectors/source-file/source_file/client.py diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index ad44e749aab36..65507376dd454 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -56,6 +56,7 @@ jobs: GH_INTEGRATION_TEST_CREDS: ${{ secrets.GH_INTEGRATION_TEST_CREDS }} GOOGLE_ANALYTICS_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_TEST_CREDS }} GOOGLE_ANALYTICS_TEST_TRACKING_ID: ${{ secrets.GOOGLE_ANALYTICS_TEST_TRACKING_ID }} + GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} GREENHOUSE_TEST_CREDS: ${{ secrets.GREENHOUSE_TEST_CREDS }} GSHEETS_INTEGRATION_TESTS_CREDS: ${{ secrets.GSHEETS_INTEGRATION_TESTS_CREDS }} HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/778daa7c-feaf-4db6-96f3-70fd645acc77.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/778daa7c-feaf-4db6-96f3-70fd645acc77.json index 8c4b0c99de4f2..be6e61d643700 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/778daa7c-feaf-4db6-96f3-70fd645acc77.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/778daa7c-feaf-4db6-96f3-70fd645acc77.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "778daa7c-feaf-4db6-96f3-70fd645acc77", "name": "File", "dockerRepository": "airbyte/source-file", - "dockerImageTag": "0.1.7", + "dockerImageTag": "0.1.8", "documentationUrl": "https://hub.docker.com/r/airbyte/source-file" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 841ae96caf1a5..693dbf8c31682 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -6,7 +6,7 @@ - sourceDefinitionId: 778daa7c-feaf-4db6-96f3-70fd645acc77 name: File dockerRepository: airbyte/source-file - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 documentationUrl: https://hub.docker.com/r/airbyte/source-file - sourceDefinitionId: fdc8b827-3257-4b33-83cc-106d234c34d4 name: Google Adwords diff --git a/airbyte-integrations/connectors/source-file/Dockerfile b/airbyte-integrations/connectors/source-file/Dockerfile index 7445ad34a5644..6bff7efa8c914 100644 --- a/airbyte-integrations/connectors/source-file/Dockerfile +++ b/airbyte-integrations/connectors/source-file/Dockerfile @@ -11,5 +11,5 @@ COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ RUN pip install ".[main]" -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-file diff --git a/airbyte-integrations/connectors/source-file/Dockerfile.test b/airbyte-integrations/connectors/source-file/Dockerfile.test index cbbfa9030bd14..4fc6b15e378f6 100644 --- a/airbyte-integrations/connectors/source-file/Dockerfile.test +++ b/airbyte-integrations/connectors/source-file/Dockerfile.test @@ -16,6 +16,6 @@ COPY $CODE_PATH $CODE_PATH COPY source_file/*.json $CODE_PATH COPY setup.py ./ -RUN pip install ".[integration_tests]" +RUN pip install ".[tests]" WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-file/build.gradle b/airbyte-integrations/connectors/source-file/build.gradle index 3c2b5939b77ad..8c3e54cf7b004 100644 --- a/airbyte-integrations/connectors/source-file/build.gradle +++ b/airbyte-integrations/connectors/source-file/build.gradle @@ -1,3 +1,5 @@ +import ru.vyarus.gradle.plugin.python.task.PythonTask + plugins { id 'airbyte-python' id 'airbyte-docker' @@ -8,6 +10,13 @@ airbytePython { moduleDirectory 'source_file' } +task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs){ + module = "pytest" + command = "-s integration_tests" +} + +integrationTest.dependsOn("customIntegrationTestPython") + dependencies { implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-file/integration_tests/__init__.py b/airbyte-integrations/connectors/source-file/integration_tests/__init__.py index 151067e91a82e..e4383c2b7b0b5 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/__init__.py @@ -22,7 +22,6 @@ SOFTWARE. """ -from .integration_source_test import TestSourceFile from .standard_source_test import SourceFileStandardTest -__all__ = ["SourceFileStandardTest", "TestSourceFile"] +__all__ = ["SourceFileStandardTest"] diff --git a/airbyte-integrations/connectors/source-file/integration_tests/client_storage_providers_test.py b/airbyte-integrations/connectors/source-file/integration_tests/client_storage_providers_test.py new file mode 100644 index 0000000000000..e651b4c016e23 --- /dev/null +++ b/airbyte-integrations/connectors/source-file/integration_tests/client_storage_providers_test.py @@ -0,0 +1,145 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import json +from pathlib import Path + +import pytest +from source_file.client import Client + +HERE = Path(__file__).parent.absolute() + + +def check_read(config, expected_columns=10, expected_rows=42): + client = Client(**config) + rows = list(client.read()) + assert len(rows) == expected_rows + assert len(rows[0]) == expected_columns + + +@pytest.mark.parametrize( + "provider_name,file_path,file_format", + [ + ("ssh", "files/test.csv", "csv"), + ("scp", "files/test.csv", "csv"), + ("sftp", "files/test.csv", "csv"), + ("ssh", "files/test.csv.gz", "csv"), # text in binary + ("ssh", "files/test.pkl", "pickle"), # binary + ("sftp", "files/test.pkl.gz", "pickle"), # binary in binary + ], +) +def test__read_from_private_ssh(provider_config, provider_name, file_path, file_format): + client = Client(dataset_name="output", format=file_format, url=file_path, provider=provider_config(provider_name)) + result = next(client.read()) + assert result == {"header1": "text", "header2": 1, "header3": 0.2} + + +@pytest.mark.parametrize( + "provider_name,file_path,file_format", + [ + ("ssh", "files/file_does_not_exist.csv", "csv"), + ("gcs", "gs://gcp-public-data-landsat/file_does_not_exist.csv", "csv"), + ], +) +def test__read_file_not_found(provider_config, provider_name, file_path, file_format): + client = Client(dataset_name="output", format=file_format, url=file_path, provider=provider_config(provider_name)) + with pytest.raises(FileNotFoundError): + next(client.read()) + + +@pytest.mark.parametrize( + "provider_name, file_path, file_format", + [ + ("ssh", "files/test.csv", "csv"), + ("ssh", "files/test.pkl", "pickle"), + ("sftp", "files/test.pkl.gz", "pickle"), + ], +) +def test__streams_from_ssh_providers(provider_config, provider_name, file_path, file_format): + client = Client(dataset_name="output", format=file_format, url=file_path, provider=provider_config(provider_name)) + streams = list(client.streams) + assert len(streams) == 1 + assert streams[0].json_schema["properties"] == { + "header1": {"type": "string"}, + "header2": {"type": "number"}, + "header3": {"type": "number"}, + } + + +@pytest.mark.parametrize( + "storage_provider, url, columns_nb, separator, has_header", + [ + # epidemiology csv + ("HTTPS", "https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv", 10, ",", True), + ("HTTPS", "storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv", 10, ",", True), + ("local", "injected by tests", 10, ",", True), + # landsat compressed csv + ("GCS", "gs://gcp-public-data-landsat/index.csv.gz", 18, ",", True), + ("GCS", "gs://gcp-public-data-landsat/index.csv.gz", 18, ",", True), + # GDELT csv + ("S3", "s3://gdelt-open-data/events/20190914.export.csv", 58, "\\t", False), + ("S3", "s3://gdelt-open-data/events/20190914.export.csv", 58, "\\t", False), + ], +) +def test__read_from_public_provider(download_gcs_public_data, storage_provider, url, columns_nb, separator, has_header): + # inject temp file path that was downloaded by the test as URL + url = download_gcs_public_data if storage_provider == "local" else url + config = { + "format": "csv", + "dataset_name": "output", + "reader_options": json.dumps({"sep": separator, "nrows": 42}), + "provider": {"storage": storage_provider}, + "url": url, + } + + check_read(config, expected_columns=columns_nb) + + +def test__read_from_private_gcs(google_cloud_service_credentials, private_google_cloud_file): + config = { + "dataset_name": "output", + "format": "csv", + "url": private_google_cloud_file, + "reader_options": json.dumps({"sep": ",", "nrows": 42}), + "provider": { + "storage": "GCS", + "service_account_json": json.dumps(google_cloud_service_credentials), + }, + } + check_read(config) + + +def test__read_from_private_aws(aws_credentials, private_aws_file): + config = { + "dataset_name": "output", + "format": "csv", + "url": private_aws_file, + "reader_options": json.dumps({"sep": ",", "nrows": 42}), + "provider": { + "storage": "S3", + "aws_access_key_id": aws_credentials["aws_access_key_id"], + "aws_secret_access_key": aws_credentials["aws_secret_access_key"], + }, + } + check_read(config) diff --git a/airbyte-integrations/connectors/source-file/integration_tests/config.json b/airbyte-integrations/connectors/source-file/integration_tests/config.json index 0eefcfbde79fb..95b85a01504c8 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/config.json +++ b/airbyte-integrations/connectors/source-file/integration_tests/config.json @@ -1,5 +1,5 @@ { - "filename": "integrationTestFile", + "dataset_name": "integrationTestFile", "format": "csv", "reader_options": "{\"sep\": \",\", \"nrows\": 20}", "url": "https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv", diff --git a/airbyte-integrations/connectors/source-file/integration_tests/catalog.json b/airbyte-integrations/connectors/source-file/integration_tests/configured_catalog.json similarity index 100% rename from airbyte-integrations/connectors/source-file/integration_tests/catalog.json rename to airbyte-integrations/connectors/source-file/integration_tests/configured_catalog.json diff --git a/airbyte-integrations/connectors/source-file/integration_tests/conftest.py b/airbyte-integrations/connectors/source-file/integration_tests/conftest.py new file mode 100644 index 0000000000000..be3746c579783 --- /dev/null +++ b/airbyte-integrations/connectors/source-file/integration_tests/conftest.py @@ -0,0 +1,184 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import json +import os +import socket +import tempfile +import uuid +from pathlib import Path +from typing import Mapping + +import boto3 +import pandas +import pytest +from botocore.errorfactory import ClientError +from google.api_core.exceptions import Conflict +from google.cloud import storage +from paramiko.client import AutoAddPolicy, SSHClient +from paramiko.ssh_exception import SSHException + +HERE = Path(__file__).parent.absolute() + + +@pytest.fixture(scope="session") +def docker_compose_file() -> Path: + return HERE / "docker-compose.yml" + + +@pytest.fixture(scope="session") +def google_cloud_service_file() -> Path: + return HERE.parent / "secrets/gcs.json" + + +@pytest.fixture(scope="session") +def google_cloud_service_credentials(google_cloud_service_file) -> Mapping: + with open(google_cloud_service_file) as json_file: + return json.load(json_file) + + +@pytest.fixture(scope="session") +def aws_credentials() -> Mapping: + filename = HERE.parent / "secrets/aws.json" + with open(filename) as json_file: + return json.load(json_file) + + +@pytest.fixture(scope="session") +def cloud_bucket_name(): + return "airbytetestbucket" + + +def is_ssh_ready(ip, port): + try: + with SSHClient() as ssh: + ssh.set_missing_host_key_policy(AutoAddPolicy) + ssh.connect( + ip, + port=port, + username="user1", + password="pass1", + ) + return True + except (SSHException, socket.error): + return False + + +@pytest.fixture(scope="session") +def ssh_service(docker_ip, docker_services): + """Ensure that SSH service is up and responsive.""" + + # `port_for` takes a container port and returns the corresponding host port + port = docker_services.port_for("ssh", 22) + docker_services.wait_until_responsive(timeout=30.0, pause=0.1, check=lambda: is_ssh_ready(docker_ip, port)) + return docker_ip + + +@pytest.fixture +def provider_config(ssh_service): + def lookup(name): + providers = { + "ssh": dict(storage="SSH", host=ssh_service, user="user1", password="pass1", port=2222), + "scp": dict(storage="SCP", host=ssh_service, user="user1", password="pass1", port=2222), + "sftp": dict(storage="SFTP", host=ssh_service, user="user1", password="pass1", port=100), + "gcs": dict(storage="GCS"), + "s3": dict(storage="S3"), + } + return providers[name] + + return lookup + + +def create_unique_gcs_bucket(storage_client, name: str) -> str: + """ + Make a unique bucket to which we'll upload the file. + (GCS buckets are part of a single global namespace.) + """ + for i in range(0, 5): + bucket_name = f"{name}-{uuid.uuid1()}" + try: + bucket = storage_client.bucket(bucket_name) + bucket.storage_class = "STANDARD" + # fixed locations are cheaper... + storage_client.create_bucket(bucket, location="us-east1") + print(f"\nNew GCS bucket created {bucket_name}") + return bucket_name + except Conflict: + print(f"\nError: {bucket_name} already exists!") + + +@pytest.fixture(scope="session") +def download_gcs_public_data(): + print("\nDownload public dataset from gcs to local /tmp") + df = pandas.read_csv("https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv") + tmp_file = tempfile.NamedTemporaryFile(delete=False) + df.to_csv(tmp_file.name, index=False) + + yield tmp_file.name + + os.remove(tmp_file.name) + print(f"\nLocal File {tmp_file.name} is now deleted") + + +@pytest.fixture(scope="session") +def private_google_cloud_file(google_cloud_service_file, cloud_bucket_name, download_gcs_public_data): + storage_client = storage.Client.from_service_account_json(str(google_cloud_service_file)) + bucket_name = create_unique_gcs_bucket(storage_client, cloud_bucket_name) + print(f"\nUpload dataset to private gcs bucket {bucket_name}") + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.blob("myfile.csv") + blob.upload_from_filename(download_gcs_public_data) + + yield f"{bucket_name}/myfile.csv" + + bucket.delete(force=True) + print(f"\nGCS Bucket {bucket_name} is now deleted") + + +@pytest.fixture(scope="session") +def private_aws_file(aws_credentials, cloud_bucket_name, download_gcs_public_data): + region = "eu-west-3" + location = {"LocationConstraint": region} + s3_client = boto3.client( + "s3", + aws_access_key_id=aws_credentials["aws_access_key_id"], + aws_secret_access_key=aws_credentials["aws_secret_access_key"], + region_name=region, + ) + bucket_name = cloud_bucket_name + print(f"\nUpload dataset to private aws bucket {bucket_name}") + try: + s3_client.head_bucket(Bucket=bucket_name) + except ClientError: + s3_client.create_bucket(Bucket=bucket_name, CreateBucketConfiguration=location) + s3_client.upload_file(download_gcs_public_data, bucket_name, "myfile.csv") + + yield f"{bucket_name}/myfile.csv" + + s3 = boto3.resource( + "s3", aws_access_key_id=aws_credentials["aws_access_key_id"], aws_secret_access_key=aws_credentials["aws_secret_access_key"] + ) + bucket = s3.Bucket(bucket_name) + bucket.objects.all().delete() + print(f"\nS3 Bucket {bucket_name} is now deleted") diff --git a/airbyte-integrations/connectors/source-file/integration_tests/docker-compose.yml b/airbyte-integrations/connectors/source-file/integration_tests/docker-compose.yml new file mode 100644 index 0000000000000..a9ec97ba6b955 --- /dev/null +++ b/airbyte-integrations/connectors/source-file/integration_tests/docker-compose.yml @@ -0,0 +1,9 @@ +version: '3' +services: + ssh: + image: atmoz/sftp + ports: + - "2222:22" + volumes: + - ./sample_files:/home/user1/files + command: user1:pass1:1001 diff --git a/airbyte-integrations/connectors/source-file/integration_tests/integration_source_test.py b/airbyte-integrations/connectors/source-file/integration_tests/integration_source_test.py deleted file mode 100644 index 85bca013debb3..0000000000000 --- a/airbyte-integrations/connectors/source-file/integration_tests/integration_source_test.py +++ /dev/null @@ -1,195 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -import json -import os -import tempfile -import uuid - -import boto3 -import pytest -from base_python import AirbyteLogger -from botocore.errorfactory import ClientError -from google.api_core.exceptions import Conflict -from google.cloud import storage -from source_file import SourceFile - - -class TestSourceFile(object): - service_account_file: str = "../secrets/gcs.json" - aws_credentials: str = "../secrets/aws.json" - cloud_bucket_name: str = "airbytetestbucket" - - @pytest.fixture(scope="class") - def download_gcs_public_data(self): - print("\nDownload public dataset from gcs to local /tmp") - config = get_config(0) - config["provider"]["storage"] = "HTTPS" - config["url"] = "https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv" - df = run_load_dataframes(config) - tmp_file = tempfile.NamedTemporaryFile(delete=False) - df.to_csv(tmp_file.name, index=False) - yield tmp_file.name - os.remove(tmp_file.name) - print(f"\nLocal File {tmp_file.name} is now deleted") - - @pytest.fixture(scope="class") - def create_gcs_private_data(self, download_gcs_public_data): - storage_client = storage.Client.from_service_account_json(self.service_account_file) - bucket_name = create_unique_gcs_bucket(storage_client, self.cloud_bucket_name) - print(f"\nUpload dataset to private gcs bucket {bucket_name}") - bucket = storage_client.get_bucket(bucket_name) - blob = bucket.blob("myfile.csv") - blob.upload_from_filename(download_gcs_public_data) - yield f"{bucket_name}/myfile.csv" - bucket.delete(force=True) - print(f"\nGCS Bucket {bucket_name} is now deleted") - - @pytest.fixture(scope="class") - def create_aws_private_data(self, download_gcs_public_data): - with open(self.aws_credentials) as json_file: - aws_config = json.load(json_file) - region = "eu-west-3" - location = {"LocationConstraint": region} - s3_client = boto3.client( - "s3", - aws_access_key_id=aws_config["aws_access_key_id"], - aws_secret_access_key=aws_config["aws_secret_access_key"], - region_name=region, - ) - bucket_name = self.cloud_bucket_name - print(f"\nUpload dataset to private aws bucket {bucket_name}") - try: - s3_client.head_bucket(Bucket=bucket_name) - except ClientError: - s3_client.create_bucket(Bucket=bucket_name, CreateBucketConfiguration=location) - s3_client.upload_file(download_gcs_public_data, bucket_name, "myfile.csv") - yield f"{bucket_name}/myfile.csv" - s3 = boto3.resource( - "s3", aws_access_key_id=aws_config["aws_access_key_id"], aws_secret_access_key=aws_config["aws_secret_access_key"] - ) - bucket = s3.Bucket(bucket_name) - bucket.objects.all().delete() - print(f"\nS3 Bucket {bucket_name} is now deleted") - - @pytest.mark.parametrize( - "reader_impl, storage_provider, url, columns_nb, config_index", - [ - # epidemiology csv - ("gcsfs", "HTTPS", "https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv", 10, 0), - ("smart_open", "HTTPS", "storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv", 10, 0), - ("smart_open", "local", "injected by tests", 10, 0), - # landsat compressed csv - ("gcsfs", "GCS", "gs://gcp-public-data-landsat/index.csv.gz", 18, 1), - ("smart_open", "GCS", "gs://gcp-public-data-landsat/index.csv.gz", 18, 0), - # GDELT csv - ("s3fs", "S3", "s3://gdelt-open-data/events/20190914.export.csv", 58, 2), - ("smart_open", "S3", "s3://gdelt-open-data/events/20190914.export.csv", 58, 2), - ], - ) - def test_public_and_local_data(self, download_gcs_public_data, reader_impl, storage_provider, url, columns_nb, config_index): - config = get_config(config_index) - config["provider"]["storage"] = storage_provider - if storage_provider != "local": - config["url"] = url - else: - # inject temp file path that was downloaded by the test as URL - config["url"] = download_gcs_public_data - config["provider"]["reader_impl"] = reader_impl - run_load_dataframes(config, expected_columns=columns_nb) - - @pytest.mark.parametrize("reader_impl", ["gcsfs", "smart_open"]) - def test_private_gcs_load(self, create_gcs_private_data, reader_impl): - config = get_config(0) - config["provider"]["storage"] = "GCS" - config["url"] = create_gcs_private_data - config["provider"]["reader_impl"] = reader_impl - with open(self.service_account_file) as json_file: - config["provider"]["service_account_json"] = json.dumps(json.load(json_file)) - run_load_dataframes(config) - - @pytest.mark.parametrize("reader_impl", ["s3fs", "smart_open"]) - def test_private_aws_load(self, create_aws_private_data, reader_impl): - config = get_config(0) - config["provider"]["storage"] = "S3" - config["url"] = create_aws_private_data - config["provider"]["reader_impl"] = reader_impl - with open(self.aws_credentials) as json_file: - aws_config = json.load(json_file) - config["provider"]["aws_access_key_id"] = aws_config["aws_access_key_id"] - config["provider"]["aws_secret_access_key"] = aws_config["aws_secret_access_key"] - run_load_dataframes(config) - - @pytest.mark.parametrize( - "storage_provider, url, user, password, host, columns_nb, rows_nb, config_index", - [ - ("SFTP", "/pub/example/readme.txt", "demo", "password", "test.rebex.net", 1, 6, 3), - ("SSH", "readme.txt", "demo", "password", "test.rebex.net", 1, 6, 3), - ], - ) - def test_private_provider(self, storage_provider, url, user, password, host, columns_nb, rows_nb, config_index): - config = get_config(config_index) - config["provider"]["storage"] = storage_provider - config["url"] = url - config["provider"]["user"] = user - config["provider"]["password"] = password - config["provider"]["host"] = host - run_load_dataframes(config, columns_nb, rows_nb) - - -def run_load_dataframes(config, expected_columns=10, expected_rows=42): - df_list = SourceFile.load_dataframes(config=config, logger=AirbyteLogger(), skip_data=False) - assert len(df_list) == 1 # Properly load 1 DataFrame - df = df_list[0] - assert len(df.columns) == expected_columns # DataFrame should have 10 columns - assert len(df.index) == expected_rows # DataFrame should have 42 rows of data - return df - - -def get_config(index: int) -> dict: - configs = [ - {"format": "csv", "reader_options": '{"sep": ",", "nrows": 42}', "provider": {}}, - {"format": "csv", "reader_options": '{"sep": ",", "nrows": 42, "compression": "gzip"}', "provider": {}}, - {"format": "csv", "reader_options": '{"sep": "\\t", "nrows": 42, "header": null}', "provider": {}}, - {"format": "csv", "reader_options": '{"sep": "\\r\\n", "names": ["text"], "header": null, "engine": "python"}', "provider": {}}, - ] - return configs[index] - - -def create_unique_gcs_bucket(storage_client, name: str) -> str: - """ - Make a unique bucket to which we'll upload the file. - (GCS buckets are part of a single global namespace.) - """ - for i in range(0, 5): - bucket_name = f"{name}-{uuid.uuid1()}" - try: - bucket = storage_client.bucket(bucket_name) - bucket.storage_class = "STANDARD" - # fixed locations are cheaper... - storage_client.create_bucket(bucket, location="us-east1") - print(f"\nNew GCS bucket created {bucket_name}") - return bucket_name - except Conflict: - print(f"\nError: {bucket_name} already exists!") diff --git a/airbyte-integrations/connectors/source-file/integration_tests/nested_config.json b/airbyte-integrations/connectors/source-file/integration_tests/nested_config.json index 56f30d01d8aef..9dee0fb49861f 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/nested_config.json +++ b/airbyte-integrations/connectors/source-file/integration_tests/nested_config.json @@ -1,5 +1,5 @@ { - "filename": "integrationTestFile", + "dataset_name": "integrationTestFile", "format": "json", "provider": { "storage": "HTTPS" }, "url": "https://think.cs.vt.edu/corgis/datasets/json/airlines/airlines.json" diff --git a/airbyte-integrations/connectors/source-file/integration_tests/sample_files/empty.csv b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/empty.csv new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.csv b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.csv new file mode 100644 index 0000000000000..743a3e6ee74a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.csv @@ -0,0 +1,3 @@ +header1,header2,header3 +text,1,0.2 +text,3,0.0 diff --git a/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.csv.gz b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.csv.gz new file mode 100644 index 0000000000000000000000000000000000000000..2a90e2cc13d31b6c9b91ab988b90ca00827902da GIT binary patch literal 60 zcmV-C0K@+uiwFo^Tnk_T19W9`bS`6ab^yyrO-xBGGStZc(?(F*n5!hUqD03~$3V{r S!ZZdl4Y&YiRaxgQ0001cMHOQJ literal 0 HcmV?d00001 diff --git a/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.pkl b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.pkl new file mode 100644 index 0000000000000000000000000000000000000000..3991ed154f25ee9ef17ab96f61490c799451d420 GIT binary patch literal 1033 zcmb7D-)j>=5Wc%4ZLJn9SZZGiKB&pdVbq5{C(5%?k6j;w$U()ukz{dSLdnqHcm*0> zd;1bwpYa5nww{&)iFSKFd?+-N0tywEAYj0xLVN3FG3tHTtG?00n+Jryw+PJHF;j5{ zips5~f*qxA1o;TSZ`#Jp_JwmIgLXrhay+SMmZs1{kP;?TFO{q}5=u+c(*(B-ABc9D zWciRwOoCJFgc*LpeQYLSQaeSyVH>%gjG2Q(?{@Iwyt=K~@FRA+-M7E!_@87~V|L** zEOQ1qlWj^~Sx^30PmxiStqa${t~sMhe9dlMC0c?QIMw^^e@-cHb!|gwkeUeFy3(UD zCXq838KKX~hz7sQH=QEBHPaI&(HZh>8}%1-JpW5a;B<8EeE;UokKJkjf&j^VP0rz5 zPX1!CGt-RWG}UybuFZJgce1Y8u}1#hHO*^UZ kKx^7SM?38oF!1Ik6vgOBrMVds3(zbrAY+BXb7ivlHk literal 0 HcmV?d00001 diff --git a/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.pkl.gz b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..063ebe370e333e6bfd2906399abd43612e94a8fc GIT binary patch literal 566 zcmV-60?GX!iwFqIOAufH19W9`bS`jfYyg#0OK;Oa5RUU|DO6C2s`OHc1CU%S0vvh( zRU!xptOnG3WsT!aHo}i+cilD;mEe%lN_*>YfPciF0%q0?N>IdO5BBHT`DW%D&wQwV z{$8uNV>iZ-#i0Fuu7p2OkP5WdUVwp@4qsyXBc5W**28qD(C*B;k1``v28s4l$Y3ax zMtkFVlK0>3m)|A5bx8a>i^N!*2<=CpDcxF_iR17U+lU~q+r|vL#4#0vb~7u|@uZ}u zMPf#v6ig^zs6;={v{6Oh5XCl}Bia>8=Ugr^s7$fzMfeF1uoZYgZHjW!HexdwvqD>a zvx^t!8P-d?yaw1_%_T!N9fHAazTy{7}VCdi$BuRd(!?+kz6<|b|KqL~4=emgepHnGh)(Hdv E07mB-ApigX literal 0 HcmV?d00001 diff --git a/airbyte-integrations/connectors/source-file/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-file/integration_tests/standard_source_test.py index 086374c1d963d..c52bfba53ce90 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/standard_source_test.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/standard_source_test.py @@ -22,27 +22,8 @@ SOFTWARE. """ -import json -import pkgutil +from base_python_test import DefaultStandardSourceTest -from airbyte_protocol import ConfiguredAirbyteCatalog, ConnectorSpecification -from base_python_test import StandardSourceTestIface - -class SourceFileStandardTest(StandardSourceTestIface): - def get_spec(self) -> ConnectorSpecification: - raw_spec = pkgutil.get_data(self.__class__.__module__.split(".")[0], "spec.json") - return ConnectorSpecification.parse_obj(json.loads(raw_spec)) - - def get_config(self) -> object: - return json.loads(pkgutil.get_data(self.__class__.__module__.split(".")[0], "config.json")) - - def get_catalog(self) -> ConfiguredAirbyteCatalog: - raw_spec = pkgutil.get_data(self.__class__.__module__.split(".")[0], "catalog.json") - return ConfiguredAirbyteCatalog.parse_obj(json.loads(raw_spec)) - - def setup(self) -> None: - pass - - def teardown(self) -> None: - pass +class SourceFileStandardTest(DefaultStandardSourceTest): + pass diff --git a/airbyte-integrations/connectors/source-file/setup.py b/airbyte-integrations/connectors/source-file/setup.py index df2a8f1587c6b..b8188fadbc908 100644 --- a/airbyte-integrations/connectors/source-file/setup.py +++ b/airbyte-integrations/connectors/source-file/setup.py @@ -24,23 +24,32 @@ from setuptools import find_packages, setup +MAIN_REQUIREMENTS = [ + "airbyte-protocol", + "base-python", + "gcsfs==0.7.1", + "genson==1.2.2", + "google-cloud-storage==1.35.0", + "pandas>=0.24.1", + "paramiko==2.7.2", + "s3fs==0.5.2", + "smart-open[all]==4.1.2", +] + +TEST_REQUIREMENTS = [ + "airbyte-python-test", + "boto3==1.16.57", + "pytest==6.1.2", + "pytest-docker==0.10.1", +] + setup( name="source_file", description="Source implementation for File", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=[ - "airbyte-protocol", - "base-python", - "gcsfs", - "genson", - "google-cloud-storage", - "pandas>=0.24.1", - "paramiko", - "s3fs", - "smart-open[all]", - ], + install_requires=MAIN_REQUIREMENTS, package_data={"": ["*.json"]}, setup_requires=["pytest-runner"], tests_require=["pytest"], @@ -49,6 +58,6 @@ # integration tests but not the main package go in integration_tests. Deps required by both should go in # install_requires. "main": [], - "integration_tests": ["airbyte-python-test", "boto3", "pytest"], + "tests": TEST_REQUIREMENTS, }, ) diff --git a/airbyte-integrations/connectors/source-file/source_file/client.py b/airbyte-integrations/connectors/source-file/source_file/client.py new file mode 100644 index 0000000000000..f03bde55e6f62 --- /dev/null +++ b/airbyte-integrations/connectors/source-file/source_file/client.py @@ -0,0 +1,341 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import json +import traceback +from typing import Iterable, List +from urllib.parse import urlparse + +import google +import numpy as np +import pandas as pd +import smart_open +from airbyte_protocol import AirbyteStream +from base_python.entrypoint import logger +from botocore import UNSIGNED +from botocore.config import Config +from genson import SchemaBuilder +from google.cloud.storage import Client as GCSClient +from google.oauth2 import service_account + + +class ConfigurationError(Exception): + """Client mis-configured""" + + +class PermissionsError(Exception): + """User don't have enough permissions""" + + +class URLFile: + """Class to manage read from file located at different providers + + Supported examples of URL this class can accept are as follows: + ``` + s3://my_bucket/my_key + s3://my_key:my_secret@my_bucket/my_key + gs://my_bucket/my_blob + hdfs:///path/file (not tested) + hdfs://path/file (not tested) + webhdfs://host:port/path/file (not tested) + ./local/path/file + ~/local/path/file + local/path/file + ./local/path/file.gz + file:///home/user/file + file:///home/user/file.bz2 + [ssh|scp|sftp]://username@host//path/file + [ssh|scp|sftp]://username@host/path/file + [ssh|scp|sftp]://username:password@host/path/file + ``` + """ + + def __init__(self, url: str, provider: dict): + self._url = url + self._provider = provider + self._file = None + + def __enter__(self): + return self._file + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + @property + def full_url(self): + return f"{self.storage_scheme}{self.url}" + + def close(self): + if self._file: + self._file.close() + self._file = None + + def open(self, binary=False): + self.close() + try: + self._file = self._open(binary=binary) + except google.api_core.exceptions.NotFound as err: + raise FileNotFoundError(self.url) from err + return self + + def _open(self, binary): + mode = "rb" if binary else "r" + storage = self.storage_scheme + url = self.url + + if storage == "gs://": + return self._open_gcs_url(binary=binary) + elif storage == "s3://": + return self._open_aws_url(binary=binary) + elif storage == "webhdfs://": + host = self._provider["host"] + port = self._provider["port"] + return smart_open.open(f"webhdfs://{host}:{port}/{url}", mode=mode) + elif storage in ("ssh://", "scp://", "sftp://"): + user = self._provider["user"] + host = self._provider["host"] + port = self._provider.get("port", 22) + # Explicitly turn off ssh keys stored in ~/.ssh + transport_params = {"connect_kwargs": {"look_for_keys": False}} + if "password" in self._provider: + password = self._provider["password"] + uri = f"{storage}{user}:{password}@{host}:{port}/{url}" + else: + uri = f"{storage}{user}@{host}:{port}/{url}" + return smart_open.open(uri, transport_params=transport_params, mode=mode) + return smart_open.open(self.full_url, mode=mode) + + @property + def url(self) -> str: + """Convert URL to remove the URL prefix (scheme) + :return: the corresponding URL without URL prefix / scheme + """ + parse_result = urlparse(self._url) + if parse_result.scheme: + return self._url.split("://")[-1] + else: + return self._url + + @property + def storage_scheme(self) -> str: + """Convert Storage Names to the proper URL Prefix + :return: the corresponding URL prefix / scheme + """ + storage_name = self._provider["storage"].upper() + parse_result = urlparse(self._url) + if storage_name == "GCS": + return "gs://" + elif storage_name == "S3": + return "s3://" + elif storage_name == "HTTPS": + return "https://" + elif storage_name == "SSH" or storage_name == "SCP": + return "scp://" + elif storage_name == "SFTP": + return "sftp://" + elif storage_name == "WEBHDFS": + return "webhdfs://" + elif storage_name == "LOCAL": + return "file://" + elif parse_result.scheme: + return parse_result.scheme + + logger.error(f"Unknown Storage provider in: {self.full_url}") + return "" + + def _open_gcs_url(self, binary) -> object: + mode = "rb" if binary else "r" + service_account_json = self._provider.get("service_account_json") + credentials = None + if service_account_json: + try: + credentials = json.loads(self._provider["service_account_json"]) + except json.decoder.JSONDecodeError as err: + error_msg = f"Failed to parse gcs service account json: {repr(err)}\n{traceback.format_exc()}" + logger.error(error_msg) + raise ConfigurationError(error_msg) from err + + if credentials: + credentials = service_account.Credentials.from_service_account_info(credentials) + client = GCSClient(credentials=credentials, project=credentials._project_id) + else: + client = GCSClient.create_anonymous_client() + file_to_close = smart_open.open(self.full_url, transport_params=dict(client=client), mode=mode) + + return file_to_close + + def _open_aws_url(self, binary): + mode = "rb" if binary else "r" + aws_access_key_id = self._provider.get("aws_access_key_id") + aws_secret_access_key = self._provider.get("aws_secret_access_key") + use_aws_account = aws_access_key_id and aws_secret_access_key + + if use_aws_account: + aws_access_key_id = self._provider.get("aws_access_key_id", "") + aws_secret_access_key = self._provider.get("aws_secret_access_key", "") + result = smart_open.open(f"{self.storage_scheme}{aws_access_key_id}:{aws_secret_access_key}@{self.url}", mode=mode) + else: + config = Config(signature_version=UNSIGNED) + params = { + "resource_kwargs": {"config": config}, + } + result = smart_open.open(self.full_url, transport_params=params, mode=mode) + return result + + +class Client: + """Class that manages reading and parsing data from streams""" + + reader_class = URLFile + + def __init__(self, dataset_name: str, url: str, provider: dict, format: str = None, reader_options: str = None): + self._dataset_name = dataset_name + self._url = url + self._provider = provider + self._reader_format = format or "csv" + self._reader_options = {} + if reader_options: + try: + self._reader_options = json.loads(reader_options) + except json.decoder.JSONDecodeError as err: + error_msg = f"Failed to parse reader options {repr(err)}\n{reader_options}\n{traceback.format_exc()}" + logger.error(error_msg) + raise ConfigurationError(error_msg) from err + + @property + def stream_name(self) -> str: + if self._dataset_name: + return self._dataset_name + return f"file_{self._provider['storage']}.{self._reader_format}" + + @staticmethod + def load_nested_json_schema(fp) -> dict: + # Use Genson Library to take JSON objects and generate schemas that describe them, + builder = SchemaBuilder() + builder.add_object(json.load(fp)) + result = builder.to_schema() + if "items" in result and "properties" in result["items"]: + result = result["items"]["properties"] + return result + + @staticmethod + def load_nested_json(fp) -> list: + result = json.load(fp) + if not isinstance(result, list): + result = [result] + return result + + def load_dataframes(self, fp, skip_data=False) -> List: + """load and return the appropriate pandas dataframe. + + :param fp: file-like object to read from + :param skip_data: limit reading data + :return: a list of dataframe loaded from files described in the configuration + """ + readers = { + # pandas.read_csv additional arguments can be passed to customize how to parse csv. + # see https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html + "csv": pd.read_csv, + # We can add option to call to pd.normalize_json to normalize semi-structured JSON data into a flat table + # by asking user to specify how to flatten the nested columns + "flat_json": pd.read_json, + "html": pd.read_html, + "excel": pd.read_excel, + "feather": pd.read_feather, + "parquet": pd.read_parquet, + "orc": pd.read_orc, + "pickle": pd.read_pickle, + } + + try: + reader = readers[self._reader_format] + except KeyError as err: + error_msg = f"Reader {self._reader_format} is not supported\n{traceback.format_exc()}" + logger.error(error_msg) + raise ConfigurationError(error_msg) from err + + reader_options = {**self._reader_options} + if skip_data and self._reader_format == "csv": + reader_options["nrows"] = 0 + reader_options["index_col"] = 0 + + return [reader(fp, **reader_options)] + + @staticmethod + def dtype_to_json_type(dtype) -> str: + """Convert Pandas Dataframe types to Airbyte Types. + + :param dtype: Pandas Dataframe type + :return: Corresponding Airbyte Type + """ + if dtype == object: + return "string" + elif dtype in ("int64", "float64"): + return "number" + elif dtype == "bool": + return "bool" + return "string" + + @property + def reader(self) -> reader_class: + return self.reader_class(url=self._url, provider=self._provider) + + @property + def binary_source(self): + binary_formats = {"excel", "feather", "parquet", "orc", "pickle"} + return self._reader_format in binary_formats + + def read(self, fields: Iterable = None) -> Iterable[dict]: + """Read data from the stream""" + with self.reader.open(binary=self.binary_source) as fp: + if self._reader_format == "json": + yield from self.load_nested_json(fp) + else: + for df in self.load_dataframes(fp): + columns = set(fields).intersection(set(df.columns)) if fields else df.columns + df = df.replace(np.nan, "NaN", regex=True) + yield from df[columns].to_dict(orient="records") + + def _stream_properties(self): + with self.reader.open(binary=self.binary_source) as fp: + if self._reader_format == "json": + return self.load_nested_json_schema(fp) + + df_list = self.load_dataframes(fp, skip_data=False) + fields = {} + for df in df_list: + for col in df.columns: + fields[col] = self.dtype_to_json_type(df[col].dtype) + return {field: {"type": fields[field]} for field in fields} + + @property + def streams(self) -> Iterable: + """Discovers available streams""" + # TODO handle discovery of directories of multiple files instead + json_schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": self._stream_properties(), + } + yield AirbyteStream(name=self.stream_name, json_schema=json_schema) diff --git a/airbyte-integrations/connectors/source-file/source_file/source.py b/airbyte-integrations/connectors/source-file/source_file/source.py index 783bd2489d3e2..882ac0b41bb2b 100644 --- a/airbyte-integrations/connectors/source-file/source_file/source.py +++ b/airbyte-integrations/connectors/source-file/source_file/source.py @@ -22,34 +22,22 @@ SOFTWARE. """ -import json -import os -import tempfile import traceback from datetime import datetime -from typing import Dict, Generator, List -from urllib.parse import urlparse +from typing import Generator, Iterable, Mapping -import gcsfs -import numpy as np -import pandas as pd from airbyte_protocol import ( AirbyteCatalog, AirbyteConnectionStatus, AirbyteMessage, AirbyteRecordMessage, - AirbyteStream, ConfiguredAirbyteCatalog, Status, Type, ) -from base_python import Source -from botocore import UNSIGNED -from botocore.config import Config -from genson import SchemaBuilder -from google.cloud.storage import Client -from s3fs import S3FileSystem -from smart_open import open +from base_python import AirbyteLogger, Source + +from .client import Client class SourceFile(Source): @@ -97,362 +85,65 @@ class SourceFile(Source): content of directories, etc in a latter iteration. """ - def check(self, logger, config: json) -> AirbyteConnectionStatus: + client_class = Client + + def _get_client(self, config: Mapping): + """Construct client""" + client = self.client_class(**config) + + return client + + def check(self, logger, config: Mapping) -> AirbyteConnectionStatus: """ Check involves verifying that the specified file is reachable with our credentials. """ - storage = SourceFile.get_storage_scheme(logger, config["provider"]["storage"], config["url"]) - url = SourceFile.get_simple_url(config["url"]) - logger.info(f"Checking access to {storage}{url}...") + client = self._get_client(config) + logger.info(f"Checking access to {client.reader.full_url}...") try: - SourceFile.open_file_url(config, logger) - return AirbyteConnectionStatus(status=Status.SUCCEEDED) + with client.reader.open(binary=client.binary_source): + return AirbyteConnectionStatus(status=Status.SUCCEEDED) except Exception as err: - reason = f"Failed to load {storage}{url}: {repr(err)}\n{traceback.format_exc()}" + reason = f"Failed to load {client.reader.full_url}: {repr(err)}\n{traceback.format_exc()}" logger.error(reason) return AirbyteConnectionStatus(status=Status.FAILED, message=reason) - def discover(self, logger, config: json) -> AirbyteCatalog: + def discover(self, logger: AirbyteLogger, config: Mapping) -> AirbyteCatalog: """ Returns an AirbyteCatalog representing the available streams and fields in this integration. For example, given valid credentials to a Remote CSV File, returns an Airbyte catalog where each csv file is a stream, and each column is a field. """ - storage = SourceFile.get_storage_scheme(logger, config["provider"]["storage"], config["url"]) - url = SourceFile.get_simple_url(config["url"]) - name = SourceFile.get_stream_name(config) - logger.info(f"Discovering schema of {name} at {storage}{url}...") - streams = [] + client = self._get_client(config) + name = client.stream_name + + logger.info(f"Discovering schema of {name} at {client.reader.full_url}...") try: - # TODO handle discovery of directories of multiple files instead - if "format" in config and config["format"] == "json": - schema = SourceFile.load_nested_json_schema(config, logger) - json_schema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": schema, - } - else: - # Don't skip data when discovering in order to infer column types - df_list = SourceFile.load_dataframes(config, logger, skip_data=False) - fields = {} - for df in df_list: - for col in df.columns: - fields[col] = SourceFile.convert_dtype(df[col].dtype) - json_schema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": {field: {"type": fields[field]} for field in fields}, - } - streams.append(AirbyteStream(name=name, json_schema=json_schema)) + streams = list(client.streams) except Exception as err: - reason = f"Failed to discover schemas of {name} at {storage}{url}: {repr(err)}\n{traceback.format_exc()}" + reason = f"Failed to discover schemas of {name} at {client.reader.full_url}: {repr(err)}\n{traceback.format_exc()}" logger.error(reason) raise err return AirbyteCatalog(streams=streams) - def read(self, logger, config: json, catalog: ConfiguredAirbyteCatalog, state: Dict[str, any]) -> Generator[AirbyteMessage, None, None]: - """ - Returns a generator of the AirbyteMessages generated by reading the source with the given configuration, catalog, and state. - """ - storage = SourceFile.get_storage_scheme(logger, config["provider"]["storage"], config["url"]) - url = SourceFile.get_simple_url(config["url"]) - name = SourceFile.get_stream_name(config) - logger.info(f"Reading {name} ({storage}{url})...") - selection = SourceFile.parse_catalog(catalog) + def read( + self, logger: AirbyteLogger, config: Mapping, catalog: ConfiguredAirbyteCatalog, state_path: Mapping[str, any] + ) -> Generator[AirbyteMessage, None, None]: + """Returns a generator of the AirbyteMessages generated by reading the source with the given configuration, catalog, and state.""" + client = self._get_client(config) + fields = self.selected_fields(catalog) + name = client.stream_name + + logger.info(f"Reading {name} ({client.reader.full_url})...") try: - if "format" in config and config["format"] == "json": - data_list = SourceFile.load_nested_json(config, logger) - for data in data_list: - yield AirbyteMessage( - type=Type.RECORD, - record=AirbyteRecordMessage(stream=name, data=data, emitted_at=int(datetime.now().timestamp()) * 1000), - ) - else: - df_list = SourceFile.load_dataframes(config, logger) - for df in df_list: - if len(selection) > 0: - columns = selection.intersection(set(df.columns)) - else: - columns = df.columns - df = df.replace(np.nan, "NaN", regex=True) - for data in df[columns].to_dict(orient="records"): - yield AirbyteMessage( - type=Type.RECORD, - record=AirbyteRecordMessage(stream=name, data=data, emitted_at=int(datetime.now().timestamp()) * 1000), - ) + for row in client.read(fields=fields): + record = AirbyteRecordMessage(stream=name, data=row, emitted_at=int(datetime.now().timestamp()) * 1000) + yield AirbyteMessage(type=Type.RECORD, record=record) except Exception as err: - reason = f"Failed to read data of {name} at {storage}{url}: {repr(err)}\n{traceback.format_exc()}" + reason = f"Failed to read data of {name} at {client.reader.full_url}: {repr(err)}\n{traceback.format_exc()}" logger.error(reason) raise err @staticmethod - def get_stream_name(config) -> str: - if "dataset_name" in config: - name = config["dataset_name"] - else: - reader_format = "csv" - if "format" in config: - reader_format = config["format"] - name = f"file_{config['provider']['storage']}.{reader_format}" - return name - - @staticmethod - def open_file_url(config, logger): - storage = SourceFile.get_storage_scheme(logger, config["provider"]["storage"], config["url"]) - url = SourceFile.get_simple_url(config["url"]) - - file_to_close = None - if storage == "gs://": - result, file_to_close = SourceFile.open_gcs_url(config, logger, storage, url) - elif storage == "s3://": - result = SourceFile.open_aws_url(config, logger, storage, url) - elif storage == "webhdfs://": - host = config["provider"]["host"] - port = config["provider"]["port"] - result = open(f"webhdfs://{host}:{port}/{url}") - elif storage == "ssh://" or storage == "scp://" or storage == "sftp://": - user = config["provider"]["user"] - host = config["provider"]["host"] - if "password" in config["provider"]: - password = config["provider"]["password"] - # Explicitly turn off ssh keys stored in ~/.ssh - transport_params = {"connect_kwargs": {"look_for_keys": False}} - result = open(f"{storage}{user}:{password}@{host}/{url}", transport_params=transport_params) - else: - result = open(f"{storage}{user}@{host}/{url}") - file_to_close = result - else: - result = open(f"{storage}{url}") - return result, file_to_close - - @staticmethod - def open_gcs_url(config, logger, storage, url): - reader_impl = SourceFile.extract_reader_impl(config) - use_gcs_service_account = "service_account_json" in config["provider"] and storage == "gs://" - file_to_close = None - if reader_impl == "gcsfs": - if use_gcs_service_account: - try: - token_dict = json.loads(config["provider"]["service_account_json"]) - except json.decoder.JSONDecodeError as err: - logger.error(f"Failed to parse gcs service account json: {repr(err)}\n{traceback.format_exc()}") - raise err - else: - token_dict = "anon" - fs = gcsfs.GCSFileSystem(token=token_dict) - file_to_close = fs.open(f"gs://{url}") - result = file_to_close - else: - if use_gcs_service_account: - try: - credentials = json.dumps(json.loads(config["provider"]["service_account_json"])) - tmp_service_account = tempfile.NamedTemporaryFile(delete=False) - with open(tmp_service_account, "w") as f: - f.write(credentials) - tmp_service_account.close() - client = Client.from_service_account_json(tmp_service_account.name) - result = open(f"gs://{url}", transport_params=dict(client=client)) - os.remove(tmp_service_account.name) - except json.decoder.JSONDecodeError as err: - logger.error(f"Failed to parse gcs service account json: {repr(err)}\n{traceback.format_exc()}") - raise err - else: - client = Client.create_anonymous_client() - result = open(f"{storage}{url}", transport_params=dict(client=client)) - return result, file_to_close - - @staticmethod - def open_aws_url(config, _, storage, url): - reader_impl = SourceFile.extract_reader_impl(config) - use_aws_account = "aws_access_key_id" in config["provider"] and "aws_secret_access_key" in config["provider"] and storage == "s3://" - if reader_impl == "s3fs": - if use_aws_account: - aws_access_key_id = None - if "aws_access_key_id" in config["provider"]: - aws_access_key_id = config["provider"]["aws_access_key_id"] - aws_secret_access_key = None - if "aws_secret_access_key" in config["provider"]: - aws_secret_access_key = config["provider"]["aws_secret_access_key"] - s3 = S3FileSystem(anon=False, key=aws_access_key_id, secret=aws_secret_access_key) - result = s3.open(f"s3://{url}", mode="r") - else: - s3 = S3FileSystem(anon=True) - result = s3.open(f"s3://{url}", mode="r") - else: - if use_aws_account: - aws_access_key_id = "" - if "aws_access_key_id" in config["provider"]: - aws_access_key_id = config["provider"]["aws_access_key_id"] - aws_secret_access_key = "" - if "aws_secret_access_key" in config["provider"]: - aws_secret_access_key = config["provider"]["aws_secret_access_key"] - result = open(f"s3://{aws_access_key_id}:{aws_secret_access_key}@{url}") - else: - config = Config(signature_version=UNSIGNED) - params = { - "resource_kwargs": {"config": config}, - } - result = open(f"{storage}{url}", transport_params=params) - return result - - @staticmethod - def extract_reader_impl(config): - # default reader impl - reader_impl = "" - if "reader_impl" in config["provider"]: - reader_impl = config["provider"]["reader_impl"] - return reader_impl - - @staticmethod - def load_nested_json_schema(config, logger) -> dict: - url, file_to_close = SourceFile.open_file_url(config, logger) - try: - # Use Genson Library to take JSON objects and generate schemas that describe them, - builder = SchemaBuilder() - builder.add_object(json.load(url)) - result = builder.to_schema() - if "items" in result and "properties" in result["items"]: - result = result["items"]["properties"] - finally: - if file_to_close: - file_to_close.close() - return result - - @staticmethod - def load_nested_json(config, logger) -> list: - url, file_to_close = SourceFile.open_file_url(config, logger) - try: - result = json.load(url) - if isinstance(result, dict): - result = [result] - finally: - if file_to_close: - file_to_close.close() - return result - - @staticmethod - def load_dataframes(config, logger, skip_data=False) -> List: - """From an Airbyte Configuration file, load and return the appropriate pandas dataframe. - - :param skip_data: limit reading data - :param config: - :param logger: - :return: a list of dataframe loaded from files described in the configuration - """ - # default format reader - reader_format = "csv" - if "format" in config: - reader_format = config["format"] - reader_options: dict = {} - if "reader_options" in config: - try: - reader_options = json.loads(config["reader_options"]) - except json.decoder.JSONDecodeError as err: - logger.error(f"Failed to parse reader options {repr(err)}\n{config['reader_options']}\n{traceback.format_exc()}") - if skip_data and reader_format == "csv": - reader_options["nrows"] = 0 - reader_options["index_col"] = 0 - url, file_to_close = SourceFile.open_file_url(config, logger) - try: - result = SourceFile.parse_file(logger, reader_format, url, reader_options) - finally: - if file_to_close: - file_to_close.close() - return result - - @staticmethod - def parse_file(logger, reader_format: str, url, reader_options: dict) -> List: - result = [] - if reader_format == "csv": - # pandas.read_csv additional arguments can be passed to customize how to parse csv. - # see https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html - result.append(pd.read_csv(url, **reader_options)) - elif reader_format == "flat_json": - # We can add option to call to pd.normalize_json to normalize semi-structured JSON data into a flat table - # by asking user to specify how to flatten the nested columns - result.append(pd.read_json(url, **reader_options)) - elif reader_format == "html": - result += pd.read_html(url, **reader_options) - elif reader_format == "excel": - result.append(pd.read_excel(url, **reader_options)) - elif reader_format == "feather": - result.append(pd.read_feather(url, **reader_options)) - elif reader_format == "parquet": - result.append(pd.read_parquet(url, **reader_options)) - elif reader_format == "orc": - result.append(pd.read_orc(url, **reader_options)) - elif reader_format == "pickle": - result.append(pd.read_pickle(url, **reader_options)) - else: - reason = f"Reader {reader_format} is not supported\n{traceback.format_exc()}" - logger.error(reason) - raise Exception(reason) - return result - - @staticmethod - def get_storage_scheme(logger, storage_name, url) -> str: - """Convert Storage Names to the proper URL Prefix - - :param logger: Logger for printing messages - :param storage_name: Name of the Storage Provider - :param url: URL of the file in case Storage is not provided but included in the URL - :return: the corresponding URL prefix / scheme - """ - storage_name = storage_name.upper() - parse_result = urlparse(url) - if storage_name == "GCS": - return "gs://" - elif storage_name == "S3": - return "s3://" - elif storage_name == "HTTPS": - return "https://" - elif storage_name == "SSH" or storage_name == "SCP": - return "scp://" - elif storage_name == "SFTP": - return "sftp://" - elif storage_name == "WEBHDFS": - return "webhdfs://" - elif storage_name == "LOCAL": - return "file://" - elif parse_result.scheme: - return parse_result.scheme - logger.error(f"Unknown Storage provider in: {storage_name} {url}") - return "" - - @staticmethod - def get_simple_url(url) -> str: - """Convert URL to remove the URL prefix (scheme) - - :param url: URL of the file - :return: the corresponding URL without URL prefix / scheme - """ - parse_result = urlparse(url) - if parse_result.scheme: - return url.split("://")[-1] - else: - return url - - @staticmethod - def convert_dtype(dtype) -> str: - """Convert Pandas Dataframe types to Airbyte Types. - - :param dtype: Pandas Dataframe type - :return: Corresponding Airbyte Type - """ - if dtype == object: - return "string" - elif dtype in ("int64", "float64"): - return "number" - elif dtype == "bool": - return "bool" - return "string" - - @staticmethod - def parse_catalog(catalog: ConfiguredAirbyteCatalog) -> set: - columns = set() + def selected_fields(catalog: ConfiguredAirbyteCatalog) -> Iterable: for configured_stream in catalog.streams: - stream = configured_stream.stream - for key in stream.json_schema["properties"].keys(): - columns.add(key) - return columns + yield from configured_stream.stream.json_schema["properties"].keys() diff --git a/airbyte-integrations/connectors/source-file/source_file/spec.json b/airbyte-integrations/connectors/source-file/source_file/spec.json index 1eddf453adc4b..c3e15ce7b9bd3 100644 --- a/airbyte-integrations/connectors/source-file/source_file/spec.json +++ b/airbyte-integrations/connectors/source-file/source_file/spec.json @@ -54,7 +54,7 @@ }, { "title": "GCS: Google Cloud Storage", - "required": ["storage", "reader_impl"], + "required": ["storage"], "properties": { "storage": { "type": "string", @@ -64,18 +64,12 @@ "service_account_json": { "type": "string", "description": "In order to access private Buckets stored on Google Cloud, this connector would need a service account json credentials with the proper permissions as described here. Please generate the credentials.json file and copy/paste its content to this field (expecting JSON formats). If accessing publicly available data, this field is not necessary." - }, - "reader_impl": { - "type": "string", - "enum": ["smart_open", "gcsfs"], - "default": "gcsfs", - "description": "This connector provides multiple methods to retrieve data from GCS using either smart-open python libraries or GCSFS" } } }, { "title": "S3: Amazon Web Services", - "required": ["storage", "reader_impl"], + "required": ["storage"], "properties": { "storage": { "type": "string", @@ -90,12 +84,6 @@ "type": "string", "description": "In order to access private Buckets stored on AWS S3, this connector would need credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", "airbyte_secret": true - }, - "reader_impl": { - "type": "string", - "enum": ["smart_open", "s3fs"], - "default": "s3fs", - "description": "This connector provides multiple methods to retrieve data from AWS S3 using either smart-open python libraries or S3FS" } } }, @@ -117,17 +105,21 @@ }, "host": { "type": "string" + }, + "port": { + "type": "number", + "default": 22 } } }, { - "title": "SFTP: Secure File Transfer Protocol", + "title": "SCP: Secure copy protocol", "required": ["storage", "user", "host"], "properties": { "storage": { "type": "string", - "enum": ["SFTP"], - "default": "SFTP" + "enum": ["SCP"], + "default": "SCP" }, "user": { "type": "string" @@ -138,24 +130,31 @@ }, "host": { "type": "string" + }, + "port": { + "type": "number", + "default": 22 } } }, { - "title": "WebHDFS: HDFS REST API (Untested)", - "required": ["storage", "host", "port"], + "title": "SFTP: Secure File Transfer Protocol", + "required": ["storage", "user", "host"], "properties": { "storage": { "type": "string", - "description": "WARNING: smart_open library provides the ability to stream files over this protocol but we haven't been able to test this as part of Airbyte yet, please use with caution. We would love to hear feedbacks from you if you are able or fail to use this!", - "enum": ["WebHDFS"], - "default": "WebHDFS" + "enum": ["SFTP"], + "default": "SFTP" }, - "host": { + "user": { "type": "string" }, - "port": { - "type": "number" + "password": { + "type": "string", + "airbyte_secret": true + }, + "host": { + "type": "string" } } }, diff --git a/airbyte-integrations/connectors/source-file/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-file/unit_tests/unit_test.py index 3d0e1af74fadf..3f83aacbb33b3 100644 --- a/airbyte-integrations/connectors/source-file/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-file/unit_tests/unit_test.py @@ -22,8 +22,8 @@ SOFTWARE. """ -from source_file.source import SourceFile +from source_file.client import Client def test_example_method(): - assert SourceFile.convert_dtype("bool") == "bool" + assert Client.dtype_to_json_type("bool") == "bool" diff --git a/docs/integrations/sources/file.md b/docs/integrations/sources/file.md index acb26c79e20de..fd559282b0de7 100644 --- a/docs/integrations/sources/file.md +++ b/docs/integrations/sources/file.md @@ -44,20 +44,17 @@ Storage Providers are mostly enabled \(and further tested\) thanks to other open | Amazon Web Services S3 | Yes | | SFTP | Yes | | SSH / SCP | Yes | -| WebHDFS | Untested | | local filesystem | Experimental | -| Azure Blob Storage | Hidden | -| HDFS | Hidden | ### File / Stream Compression | Compression | Supported? | | :--- | :--- | | Gzip | Yes | -| Zip | Untested | -| Bzip2 | Untested | -| Lzma | Untested | -| Xz | Untested | +| Zip | No | +| Bzip2 | No | +| Lzma | No | +| Xz | No | | Snappy | No | ### File Formats @@ -72,14 +69,10 @@ File Formats are mostly enabled \(and further tested\) thanks to other open-sour | JSON | Experimental | | HTML | Untested | | XML | No | -| Excel | Untested | -| Feather | Untested | -| Parquet | Untested | -| Orc | Untested | -| Pickle | Untested | -| SAS | Hidden | -| SPS | Hidden | -| Stata | Hidden | +| Excel | Yes | +| Feather | Yes | +| Parquet | Yes | +| Pickle | Yes | ### Performance considerations diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 5b856e027c99a..68d3fff9aa27b 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -21,7 +21,7 @@ write_standard_creds destination-bigquery "$BIGQUERY_INTEGRATION_TEST_CREDS" "cr write_standard_creds destination-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" write_standard_creds destination-redshift "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS" -write_standard_creds source-file "$BIGQUERY_INTEGRATION_TEST_CREDS" "gcs.json" +write_standard_creds source-file "$GOOGLE_CLOUD_STORAGE_TEST_CREDS" "gcs.json" write_standard_creds source-braintree-singer "$BRAINTREE_TEST_CREDS" write_standard_creds source-drift "$DRIFT_INTEGRATION_TEST_CREDS" write_standard_creds source-file "$AWS_S3_INTEGRATION_TEST_CREDS" "aws.json" From 8f2df32fabe58f04c78063a6e53060447e31fc24 Mon Sep 17 00:00:00 2001 From: Artem Astapenko <3767150+Jamakase@users.noreply.github.com> Date: Thu, 28 Jan 2021 04:36:57 +0300 Subject: [PATCH 4/8] Add new api call for check connection on update (#1870) --- .../src/components/hooks/services/useDestinationHook.tsx | 9 ++++----- .../src/components/hooks/services/useSourceHook.tsx | 7 +++---- airbyte-webapp/src/core/resources/Scheduler.ts | 4 ++++ .../components/DestinationSettings.tsx | 3 +-- .../pages/SourceItemPage/components/SourceSettings.tsx | 3 +-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/airbyte-webapp/src/components/hooks/services/useDestinationHook.tsx b/airbyte-webapp/src/components/hooks/services/useDestinationHook.tsx index 050784aee5973..ef1cc0636386e 100644 --- a/airbyte-webapp/src/components/hooks/services/useDestinationHook.tsx +++ b/airbyte-webapp/src/components/hooks/services/useDestinationHook.tsx @@ -133,16 +133,15 @@ const useDestination = () => { const updateDestination = async ({ values, - destinationId, - destinationDefinitionId + destinationId }: { values: ValuesProps; destinationId: string; - destinationDefinitionId: string; }) => { await destinationCheckConnectionShape({ - destinationDefinitionId, - connectionConfiguration: values.connectionConfiguration + connectionConfiguration: values.connectionConfiguration, + name: values.name, + destinationId }); return await updatedestination( diff --git a/airbyte-webapp/src/components/hooks/services/useSourceHook.tsx b/airbyte-webapp/src/components/hooks/services/useSourceHook.tsx index 2a14c725e3585..f85959ecb85ef 100644 --- a/airbyte-webapp/src/components/hooks/services/useSourceHook.tsx +++ b/airbyte-webapp/src/components/hooks/services/useSourceHook.tsx @@ -117,15 +117,14 @@ const useSource = () => { const updateSource = async ({ values, - sourceId, - sourceDefinitionId + sourceId }: { values: ValuesProps; sourceId: string; - sourceDefinitionId: string; }) => { await sourceCheckConnectionShape({ - sourceDefinitionId, + name: values.name, + sourceId, connectionConfiguration: values.connectionConfiguration }); diff --git a/airbyte-webapp/src/core/resources/Scheduler.ts b/airbyte-webapp/src/core/resources/Scheduler.ts index 6d00deac35bde..85088dd1a2c18 100644 --- a/airbyte-webapp/src/core/resources/Scheduler.ts +++ b/airbyte-webapp/src/core/resources/Scheduler.ts @@ -40,6 +40,8 @@ export default class SchedulerResource extends BaseResource fetch: async (params: any): Promise => { const url = !params.sourceId ? `${this.url(params)}/sources/check_connection` + : params.connectionConfiguration + ? `${super.rootUrl()}sources/check_connection_for_update` : `${super.rootUrl()}sources/check_connection`; const result = await this.fetch("post", url, params); @@ -76,6 +78,8 @@ export default class SchedulerResource extends BaseResource fetch: async (params: any): Promise => { const url = !params.destinationId ? `${this.url(params)}/destinations/check_connection` + : params.connectionConfiguration + ? `${super.rootUrl()}destinations/check_connection_for_update` : `${super.rootUrl()}destinations/check_connection`; const result = await this.fetch("post", url, params); diff --git a/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/components/DestinationSettings.tsx b/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/components/DestinationSettings.tsx index c41bc2700f0c2..1a457f3099c83 100644 --- a/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/components/DestinationSettings.tsx +++ b/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/components/DestinationSettings.tsx @@ -53,8 +53,7 @@ const DestinationsSettings: React.FC = ({ try { await updateDestination({ values, - destinationId: currentDestination.destinationId, - destinationDefinitionId: currentDestination.destinationDefinitionId + destinationId: currentDestination.destinationId }); setSaved(true); diff --git a/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SourceSettings.tsx b/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SourceSettings.tsx index d21cf3f3cb496..af4a06eb59523 100644 --- a/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SourceSettings.tsx +++ b/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SourceSettings.tsx @@ -52,8 +52,7 @@ const SourceSettings: React.FC = ({ try { await updateSource({ values, - sourceId: currentSource.sourceId, - sourceDefinitionId: currentSource.sourceDefinitionId + sourceId: currentSource.sourceId }); setSaved(true); From 9563b297cccc5dfd5dff6e08208b03cfbc84fcf0 Mon Sep 17 00:00:00 2001 From: "Sherif A. Nada" Date: Wed, 27 Jan 2021 17:49:50 -0800 Subject: [PATCH 5/8] Source shopify: coerce union type to string (#1867) --- .../source_shopify_singer/source.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-shopify-singer/source_shopify_singer/source.py b/airbyte-integrations/connectors/source-shopify-singer/source_shopify_singer/source.py index 657d15863e7a6..4f4aad0a9aef0 100644 --- a/airbyte-integrations/connectors/source-shopify-singer/source_shopify_singer/source.py +++ b/airbyte-integrations/connectors/source-shopify-singer/source_shopify_singer/source.py @@ -25,7 +25,7 @@ import json import shopify -from airbyte_protocol import AirbyteConnectionStatus, Status +from airbyte_protocol import AirbyteCatalog, AirbyteConnectionStatus, Status from base_python import AirbyteLogger from base_singer import SingerSource @@ -58,6 +58,20 @@ def check_config(self, logger: AirbyteLogger, config_path: str, config: json) -> def discover_cmd(self, logger: AirbyteLogger, config_path: str) -> str: return f"{TAP_CMD} -c {config_path} --discover" + def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: + catalog: AirbyteCatalog = super().discover(logger, config) + + # the metafields stream has a union type i.e: ["null", "integer", "object", "string"], but Airbyte doesn't currently allow the notion of a + # union type. See https://github.com/airbytehq/airbyte/issues/1864 for more details. + # For now we always declare this field is a nullable string as an escape hatch. + for stream in catalog.streams: + if stream.name == "metafields": + schema = stream.json_schema + if "properties" in schema and "value" in schema["properties"]: + schema["properties"]["value"]["type"] = ["null", "string"] + + return catalog + def read_cmd(self, logger: AirbyteLogger, config_path: str, catalog_path: str, state_path: str = None) -> str: state_path = f"--state {state_path}" if state_path else "" return f"{TAP_CMD} -c {config_path} --catalog {catalog_path} {state_path}" From 38c1a8fa01ad5432ae0692ef17db5f513f28bbb5 Mon Sep 17 00:00:00 2001 From: Yevhenii <34103125+yevhenii-ldv@users.noreply.github.com> Date: Thu, 28 Jan 2021 05:10:55 +0200 Subject: [PATCH 6/8] #1763 Issue: adopt best practice for Shopify Source (#1817) Co-authored-by: Sherif Nada --- .../b1892b11-788d-44bd-b9ec-3a436f7b54ce.json | 2 +- .../resources/seed/source_definitions.yaml | 2 +- .../source-shopify-singer/Dockerfile | 2 +- .../integration_tests/configured_catalog.json | 3506 +++++++++++++++++ .../integration_test_catalog.json | 260 -- .../integration_tests/standard_source_test.py | 28 +- .../source_shopify_singer/source.py | 44 +- 7 files changed, 3530 insertions(+), 314 deletions(-) create mode 100644 airbyte-integrations/connectors/source-shopify-singer/integration_tests/configured_catalog.json delete mode 100644 airbyte-integrations/connectors/source-shopify-singer/integration_tests/integration_test_catalog.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b1892b11-788d-44bd-b9ec-3a436f7b54ce.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b1892b11-788d-44bd-b9ec-3a436f7b54ce.json index 06bb172e5d620..26bb64e71b92a 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b1892b11-788d-44bd-b9ec-3a436f7b54ce.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b1892b11-788d-44bd-b9ec-3a436f7b54ce.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "b1892b11-788d-44bd-b9ec-3a436f7b54ce", "name": "Shopify", "dockerRepository": "airbyte/source-shopify-singer", - "dockerImageTag": "0.1.6", + "dockerImageTag": "0.1.7", "documentationUrl": "https://hub.docker.com/r/airbyte/source-shopify-singer" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 693dbf8c31682..52c6c0dd9b3db 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -86,7 +86,7 @@ - sourceDefinitionId: b1892b11-788d-44bd-b9ec-3a436f7b54ce name: Shopify dockerRepository: airbyte/source-shopify-singer - dockerImageTag: 0.1.6 + dockerImageTag: 0.1.7 documentationUrl: https://hub.docker.com/r/airbyte/source-shopify-singer - sourceDefinitionId: 9845d17a-45f1-4070-8a60-50914b1c8e2b name: HTTP Request diff --git a/airbyte-integrations/connectors/source-shopify-singer/Dockerfile b/airbyte-integrations/connectors/source-shopify-singer/Dockerfile index 3210040d9c0c5..ded35f10c1155 100644 --- a/airbyte-integrations/connectors/source-shopify-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-shopify-singer/Dockerfile @@ -13,5 +13,5 @@ COPY setup.py ./ RUN pip install tap-shopify==1.2.6 RUN pip install ".[main]" -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.7 LABEL io.airbyte.name=airbyte/source-shopify-singer diff --git a/airbyte-integrations/connectors/source-shopify-singer/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-shopify-singer/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..d0038b75c6fb1 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify-singer/integration_tests/configured_catalog.json @@ -0,0 +1,3506 @@ +{ + "streams": [ + { + "stream": { + "name": "orders", + "json_schema": { + "properties": { + "presentment_currency": { + "type": ["null", "string"] + }, + "subtotal_price_set": {}, + "total_discounts_set": {}, + "total_line_items_price_set": {}, + "total_price_set": {}, + "total_shipping_price_set": {}, + "total_tax_set": {}, + "total_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "line_items": { + "items": { + "properties": { + "applied_discounts": {}, + "total_discount_set": {}, + "pre_tax_price_set": {}, + "price_set": {}, + "grams": { + "type": ["null", "integer"] + }, + "compare_at_price": { + "type": ["null", "string"] + }, + "destination_location_id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "line_price": { + "type": ["null", "string"] + }, + "origin_location_id": { + "type": ["null", "integer"] + }, + "applied_discount": { + "type": ["null", "integer"] + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "properties": { + "anyOf": [ + { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + { + "properties": {}, + "type": ["null", "object"] + } + ] + }, + "tax_code": { + "type": ["null", "string"] + }, + "discount_allocations": { + "items": { + "properties": { + "discount_application_index": { + "type": ["null", "integer"] + }, + "amount_set": {}, + "amount": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "pre_tax_price": { + "type": ["null", "number"] + }, + "sku": { + "type": ["null", "string"] + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "total_discount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "name": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "origin_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "destination_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "processing_method": { + "type": ["null", "string"] + }, + "order_number": { + "type": ["null", "integer"] + }, + "confirmed": { + "type": ["null", "boolean"] + }, + "total_discounts": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "total_line_items_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "order_adjustments": { + "items": { + "properties": { + "order_id": { + "type": ["null", "integer"] + }, + "tax_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "refund_id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "kind": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "reason": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "shipping_lines": { + "items": { + "properties": { + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "phone": { + "type": ["null", "string"] + }, + "discounted_price_set": {}, + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "discount_allocations": { + "items": { + "properties": { + "discount_application_index": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "delivery_category": { + "type": ["null", "string"] + }, + "discounted_price": { + "type": ["null", "number"] + }, + "code": { + "type": ["null", "string"] + }, + "requested_fulfillment_service_id": { + "type": ["null", "string"] + }, + "carrier_identifier": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "device_id": { + "type": ["null", "integer"] + }, + "cancel_reason": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "payment_gateway_names": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "source_identifier": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "processed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "referring_site": { + "type": ["null", "string"] + }, + "contact_email": { + "type": ["null", "string"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "fulfillments": { + "items": { + "properties": { + "location_id": { + "type": ["null", "integer"] + }, + "receipt": { + "type": ["null", "object"], + "properties": { + "testcase": { + "type": ["null", "boolean"] + }, + "authorization": { + "type": ["null", "string"] + } + } + }, + "tracking_number": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "shipment_status": { + "type": ["null", "string"] + }, + "line_items": { + "items": { + "properties": { + "applied_discounts": {}, + "total_discount_set": {}, + "pre_tax_price_set": {}, + "price_set": {}, + "grams": { + "type": ["null", "integer"] + }, + "compare_at_price": { + "type": ["null", "string"] + }, + "destination_location_id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "line_price": { + "type": ["null", "string"] + }, + "origin_location_id": { + "type": ["null", "integer"] + }, + "applied_discount": { + "type": ["null", "integer"] + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "properties": { + "anyOf": [ + { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + { + "properties": {}, + "type": ["null", "object"] + } + ] + }, + "tax_code": { + "type": ["null", "string"] + }, + "discount_allocations": { + "items": { + "properties": { + "discount_application_index": { + "type": ["null", "integer"] + }, + "amount_set": {}, + "amount": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "pre_tax_price": { + "type": ["null", "number"] + }, + "sku": { + "type": ["null", "string"] + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "total_discount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "name": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "origin_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "destination_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "tracking_url": { + "type": ["null", "string"] + }, + "service": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "tracking_urls": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "tracking_numbers": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "id": { + "type": ["null", "integer"] + }, + "tracking_company": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "customer": { + "type": "object", + "properties": { + "last_order_name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "multipass_identifier": { + "type": ["null", "string"] + }, + "default_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + }, + "orders_count": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "verified_email": { + "type": ["null", "boolean"] + }, + "total_spent": { + "type": ["null", "string"] + }, + "last_order_id": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "note": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "addresses": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + } + }, + "last_name": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "tax_exempt": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "accepts_marketing": { + "type": ["null", "boolean"] + }, + "accepts_marketing_updated_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "test": { + "type": ["null", "boolean"] + }, + "total_tax": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "payment_details": { + "properties": { + "avs_result_code": { + "type": ["null", "string"] + }, + "credit_card_company": { + "type": ["null", "string"] + }, + "cvv_result_code": { + "type": ["null", "string"] + }, + "credit_card_bin": { + "type": ["null", "string"] + }, + "credit_card_number": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "number": { + "type": ["null", "integer"] + }, + "email": { + "type": ["null", "string"] + }, + "source_name": { + "type": ["null", "string"] + }, + "landing_site_ref": { + "type": ["null", "string"] + }, + "shipping_address": { + "properties": { + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "longitude": { + "type": ["null", "number"] + }, + "address2": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "total_price_usd": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "closed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "discount_applications": { + "items": { + "properties": { + "target_type": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "target_selection": { + "type": ["null", "string"] + }, + "allocation_method": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "name": { + "type": ["null", "string"] + }, + "note": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "source_url": { + "type": ["null", "string"] + }, + "subtotal_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "billing_address": { + "properties": { + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "longitude": { + "type": ["null", "number"] + }, + "address2": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "landing_site": { + "type": ["null", "string"] + }, + "taxes_included": { + "type": ["null", "boolean"] + }, + "token": { + "type": ["null", "string"] + }, + "app_id": { + "type": ["null", "integer"] + }, + "total_tip_received": { + "type": ["null", "string"] + }, + "browser_ip": { + "type": ["null", "string"] + }, + "discount_codes": { + "items": { + "properties": { + "code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "phone": { + "type": ["null", "string"] + }, + "note_attributes": { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "order_status_url": { + "type": ["null", "string"] + }, + "client_details": { + "properties": { + "session_hash": { + "type": ["null", "string"] + }, + "accept_language": { + "type": ["null", "string"] + }, + "browser_width": { + "type": ["null", "integer"] + }, + "user_agent": { + "type": ["null", "string"] + }, + "browser_ip": { + "type": ["null", "string"] + }, + "browser_height": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "buyer_accepts_marketing": { + "type": ["null", "boolean"] + }, + "checkout_token": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "financial_status": { + "type": ["null", "string"] + }, + "customer_locale": { + "type": ["null", "string"] + }, + "checkout_id": { + "type": ["null", "integer"] + }, + "total_weight": { + "type": ["null", "integer"] + }, + "gateway": { + "type": ["null", "string"] + }, + "cart_token": { + "type": ["null", "string"] + }, + "cancelled_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "refunds": { + "items": { + "properties": { + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "refund_line_items": { + "items": { + "properties": { + "line_item": { + "properties": { + "applied_discounts": {}, + "total_discount_set": {}, + "pre_tax_price_set": {}, + "price_set": {}, + "grams": { + "type": ["null", "integer"] + }, + "compare_at_price": { + "type": ["null", "string"] + }, + "destination_location_id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "line_price": { + "type": ["null", "string"] + }, + "origin_location_id": { + "type": ["null", "integer"] + }, + "applied_discount": { + "type": ["null", "integer"] + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "properties": { + "anyOf": [ + { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + { + "properties": {}, + "type": ["null", "object"] + } + ] + }, + "tax_code": { + "type": ["null", "string"] + }, + "discount_allocations": { + "items": { + "properties": { + "discount_application_index": { + "type": ["null", "integer"] + }, + "amount_set": {}, + "amount": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "pre_tax_price": { + "type": ["null", "number"] + }, + "sku": { + "type": ["null", "string"] + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "total_discount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "name": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "origin_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "destination_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "line_item_id": { + "type": ["null", "integer"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "total_tax": { + "type": ["null", "number"] + }, + "restock_type": { + "type": ["null", "string"] + }, + "subtotal": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "restock": { + "type": ["null", "boolean"] + }, + "note": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "processed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "order_adjustments": { + "items": { + "properties": { + "order_id": { + "type": ["null", "integer"] + }, + "tax_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "refund_id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "kind": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "reason": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "reference": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "abandoned_checkouts", + "json_schema": { + "type": "object", + "properties": { + "note_attributes": {}, + "location_id": { + "type": ["null", "integer"] + }, + "buyer_accepts_marketing": { + "type": ["null", "boolean"] + }, + "currency": { + "type": ["null", "string"] + }, + "completed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "token": { + "type": ["null", "string"] + }, + "billing_address": { + "type": ["null", "object"], + "properties": { + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "zip": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "longitude": { + "type": ["null", "number"] + } + } + }, + "email": { + "type": ["null", "string"] + }, + "discount_codes": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "code": { + "type": ["null", "string"] + } + } + } + }, + "customer_locale": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "gateway": { + "type": ["null", "string"] + }, + "referring_site": { + "type": ["null", "string"] + }, + "source_identifier": { + "type": ["null", "string"] + }, + "total_weight": { + "type": ["null", "integer"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "total_line_items_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "closed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "device_id": { + "type": ["null", "integer"] + }, + "phone": { + "type": ["null", "string"] + }, + "source_name": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "total_tax": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "subtotal_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "line_items": { + "items": { + "properties": { + "applied_discounts": {}, + "total_discount_set": {}, + "pre_tax_price_set": {}, + "price_set": {}, + "grams": { + "type": ["null", "integer"] + }, + "compare_at_price": { + "type": ["null", "string"] + }, + "destination_location_id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "line_price": { + "type": ["null", "string"] + }, + "origin_location_id": { + "type": ["null", "integer"] + }, + "applied_discount": { + "type": ["null", "integer"] + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "properties": { + "anyOf": [ + { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + { + "properties": {}, + "type": ["null", "object"] + } + ] + }, + "tax_code": { + "type": ["null", "string"] + }, + "discount_allocations": { + "items": { + "properties": { + "discount_application_index": { + "type": ["null", "integer"] + }, + "amount_set": {}, + "amount": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "pre_tax_price": { + "type": ["null", "number"] + }, + "sku": { + "type": ["null", "string"] + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "total_discount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "name": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "origin_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "destination_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "source_url": { + "type": ["null", "string"] + }, + "total_discounts": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "note": { + "type": ["null", "string"] + }, + "presentment_currency": { + "type": ["null", "string"] + }, + "shipping_lines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "applied_discounts": {}, + "custom_tax_lines": {}, + "phone": { + "type": ["null", "string"] + }, + "validation_context": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "carrier_identifier": { + "type": ["null", "string"] + }, + "api_client_id": { + "type": ["null", "integer"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requested_fulfillment_service_id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "carrier_service_id": { + "type": ["null", "integer"] + }, + "delivery_category": { + "type": ["null", "string"] + }, + "markup": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + } + } + } + }, + "user_id": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "shipping_address": { + "type": ["null", "object"], + "properties": { + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "zip": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "longitude": { + "type": ["null", "number"] + } + } + }, + "abandoned_checkout_url": { + "type": ["null", "string"] + }, + "landing_site": { + "type": ["null", "string"] + }, + "customer": { + "type": "object", + "properties": { + "last_order_name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "multipass_identifier": { + "type": ["null", "string"] + }, + "default_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + }, + "orders_count": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "verified_email": { + "type": ["null", "boolean"] + }, + "total_spent": { + "type": ["null", "string"] + }, + "last_order_id": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "note": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "addresses": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + } + }, + "last_name": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "tax_exempt": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "accepts_marketing": { + "type": ["null", "boolean"] + }, + "accepts_marketing_updated_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "total_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "cart_token": { + "type": ["null", "string"] + }, + "taxes_included": { + "type": ["null", "boolean"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "metafields", + "json_schema": { + "properties": { + "owner_id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "owner_resource": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "namespace": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "integer", "object", "string"], + "properties": {} + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + }, + "type": "object" + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "customers", + "json_schema": { + "type": "object", + "properties": { + "last_order_name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "multipass_identifier": { + "type": ["null", "string"] + }, + "default_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + }, + "orders_count": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "verified_email": { + "type": ["null", "boolean"] + }, + "total_spent": { + "type": ["null", "string"] + }, + "last_order_id": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "note": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "addresses": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + } + }, + "last_name": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "tax_exempt": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "accepts_marketing": { + "type": ["null", "boolean"] + }, + "accepts_marketing_updated_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "products", + "json_schema": { + "properties": { + "published_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "published_scope": { + "type": ["null", "string"] + }, + "vendor": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "body_html": { + "type": ["null", "string"] + }, + "product_type": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "options": { + "type": ["null", "array"], + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "values": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "id": { + "type": ["null", "integer"] + }, + "position": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + } + }, + "image": { + "properties": { + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "variant_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "height": { + "type": ["null", "integer"] + }, + "alt": { + "type": ["null", "string"] + }, + "src": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "width": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "handle": { + "type": ["null", "string"] + }, + "images": { + "type": ["null", "array"], + "items": { + "properties": { + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "variant_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "height": { + "type": ["null", "integer"] + }, + "alt": { + "type": ["null", "string"] + }, + "src": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "width": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + } + }, + "template_suffix": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "variants": { + "type": ["null", "array"], + "items": { + "properties": { + "barcode": { + "type": ["null", "string"] + }, + "tax_code": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "weight_unit": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "position": { + "type": ["null", "integer"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "image_id": { + "type": ["null", "integer"] + }, + "inventory_policy": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "inventory_item_id": { + "type": ["null", "integer"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "weight": { + "type": ["null", "number"] + }, + "inventory_management": { + "type": ["null", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "option1": { + "type": ["null", "string"] + }, + "compare_at_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "option2": { + "type": ["null", "string"] + }, + "old_inventory_quantity": { + "type": ["null", "integer"] + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "inventory_quantity": { + "type": ["null", "integer"] + }, + "grams": { + "type": ["null", "integer"] + }, + "option3": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + } + }, + "type": "object" + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "custom_collections", + "json_schema": { + "properties": { + "handle": { + "type": ["null", "string"] + }, + "sort_order": { + "type": ["null", "string"] + }, + "body_html": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "published_scope": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "image": { + "properties": { + "alt": { + "type": ["null", "string"] + }, + "src": { + "type": ["null", "string"] + }, + "width": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"] + }, + "height": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "published_at": { + "type": ["null", "string"] + }, + "template_suffix": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "collects", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "collection_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "position": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "sort_value": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "order_refunds", + "json_schema": { + "type": "object", + "properties": { + "order_id": { + "type": ["null", "integer"] + }, + "restock": { + "type": ["null", "boolean"] + }, + "order_adjustments": { + "items": { + "properties": { + "order_id": { + "type": ["null", "integer"] + }, + "tax_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "refund_id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "kind": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "reason": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "processed_at": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "note": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "refund_line_items": { + "type": ["null", "array"], + "items": { + "properties": { + "location_id": { + "type": ["null", "integer"] + }, + "subtotal_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "total_tax_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "line_item_id": { + "type": ["null", "integer"] + }, + "total_tax": { + "type": ["null", "number"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "line_item": { + "properties": { + "gift_card": { + "type": ["null", "boolean"] + }, + "price": { + "type": ["null", "string"] + }, + "tax_lines": { + "type": ["null", "array"], + "items": { + "properties": { + "price_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "price": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + } + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "properties": { + "type": ["null", "array"], + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "quantity": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + }, + "grams": { + "type": ["null", "integer"] + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "vendor": { + "type": ["null", "string"] + }, + "price_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "pre_tax_price": { + "type": ["null", "string"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "total_discount_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "discount_allocations": { + "type": ["null", "array"], + "items": { + "properties": { + "amount": { + "type": ["null", "string"] + }, + "amount_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "discount_application_index": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + } + }, + "pre_tax_price_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "total_discount": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "subtotal": { + "type": ["null", "number"] + }, + "restock_type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "transactions", + "json_schema": { + "properties": { + "error_code": { + "type": ["null", "string"] + }, + "device_id": { + "type": ["null", "integer"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "parent_id": { + "type": ["null", "integer"] + }, + "test": { + "type": ["null", "boolean"] + }, + "kind": { + "type": ["null", "string"] + }, + "order_id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "authorization": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "source_name": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "payment_details": { + "properties": { + "cvv_result_code": { + "type": ["null", "string"] + }, + "credit_card_bin": { + "type": ["null", "string"] + }, + "credit_card_company": { + "type": ["null", "string"] + }, + "credit_card_number": { + "type": ["null", "string"] + }, + "avs_result_code": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "gateway": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "receipt": { + "type": ["null", "object"], + "properties": { + "fee_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "gross_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "tax_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + } + }, + "patternProperties": { + ".+": {} + } + }, + "location_id": { + "type": ["null", "integer"] + } + }, + "type": "object" + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-shopify-singer/integration_tests/integration_test_catalog.json b/airbyte-integrations/connectors/source-shopify-singer/integration_tests/integration_test_catalog.json deleted file mode 100644 index 9493329271da9..0000000000000 --- a/airbyte-integrations/connectors/source-shopify-singer/integration_tests/integration_test_catalog.json +++ /dev/null @@ -1,260 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "products", - "json_schema": { - "properties": { - "published_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "published_scope": { - "type": ["null", "string"] - }, - "vendor": { - "type": ["null", "string"] - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "body_html": { - "type": ["null", "string"] - }, - "product_type": { - "type": ["null", "string"] - }, - "tags": { - "type": ["null", "string"] - }, - "options": { - "type": ["null", "array"], - "items": { - "properties": { - "name": { - "type": ["null", "string"] - }, - "product_id": { - "type": ["null", "integer"] - }, - "values": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "id": { - "type": ["null", "integer"] - }, - "position": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - } - }, - "image": { - "properties": { - "updated_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "variant_ids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "height": { - "type": ["null", "integer"] - }, - "alt": { - "type": ["null", "string"] - }, - "src": { - "type": ["null", "string"] - }, - "position": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "admin_graphql_api_id": { - "type": ["null", "string"] - }, - "width": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "handle": { - "type": ["null", "string"] - }, - "images": { - "type": ["null", "array"], - "items": { - "properties": { - "updated_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "variant_ids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "height": { - "type": ["null", "integer"] - }, - "alt": { - "type": ["null", "string"] - }, - "src": { - "type": ["null", "string"] - }, - "position": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "admin_graphql_api_id": { - "type": ["null", "string"] - }, - "width": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - } - }, - "template_suffix": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "variants": { - "type": ["null", "array"], - "items": { - "properties": { - "barcode": { - "type": ["null", "string"] - }, - "tax_code": { - "type": ["null", "string"] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "weight_unit": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "position": { - "type": ["null", "integer"] - }, - "price": { - "type": ["null", "number"], - "multipleOf": 1e-10 - }, - "image_id": { - "type": ["null", "integer"] - }, - "inventory_policy": { - "type": ["null", "string"] - }, - "sku": { - "type": ["null", "string"] - }, - "inventory_item_id": { - "type": ["null", "integer"] - }, - "fulfillment_service": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "weight": { - "type": ["null", "number"] - }, - "inventory_management": { - "type": ["null", "string"] - }, - "taxable": { - "type": ["null", "boolean"] - }, - "admin_graphql_api_id": { - "type": ["null", "string"] - }, - "option1": { - "type": ["null", "string"] - }, - "compare_at_price": { - "type": ["null", "number"], - "multipleOf": 1e-10 - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "option2": { - "type": ["null", "string"] - }, - "old_inventory_quantity": { - "type": ["null", "integer"] - }, - "requires_shipping": { - "type": ["null", "boolean"] - }, - "inventory_quantity": { - "type": ["null", "integer"] - }, - "grams": { - "type": ["null", "integer"] - }, - "option3": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - } - }, - "admin_graphql_api_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - } - }, - "type": "object" - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updated_at"] - }, - "sync_mode": "incremental", - "cursor_field": ["updated_at"] - } - ] -} diff --git a/airbyte-integrations/connectors/source-shopify-singer/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-shopify-singer/integration_tests/standard_source_test.py index 8aa9db67a3f9e..4763ef41006d7 100644 --- a/airbyte-integrations/connectors/source-shopify-singer/integration_tests/standard_source_test.py +++ b/airbyte-integrations/connectors/source-shopify-singer/integration_tests/standard_source_test.py @@ -22,30 +22,8 @@ SOFTWARE. """ -import json -import pkgutil +from base_python_test import DefaultStandardSourceTest -from airbyte_protocol import ConfiguredAirbyteCatalog, ConnectorSpecification -from base_python_test import StandardSourceTestIface - -class SourceShopifySingerStandardTest(StandardSourceTestIface): - def __init__(self): - pass - - def get_spec(self) -> ConnectorSpecification: - raw_spec = pkgutil.get_data(self.__class__.__module__.split(".")[0], "spec.json") - return ConnectorSpecification.parse_obj(json.loads(raw_spec)) - - def get_config(self) -> object: - return json.loads(pkgutil.get_data(self.__class__.__module__.split(".")[0], "config.json")) - - def get_catalog(self) -> ConfiguredAirbyteCatalog: - raw_catalog = pkgutil.get_data(self.__class__.__module__.split(".")[0], "integration_test_catalog.json") - return ConfiguredAirbyteCatalog.parse_obj(json.loads(raw_catalog)) - - def setup(self) -> None: - pass - - def teardown(self) -> None: - pass +class SourceShopifySingerStandardTest(DefaultStandardSourceTest): + pass diff --git a/airbyte-integrations/connectors/source-shopify-singer/source_shopify_singer/source.py b/airbyte-integrations/connectors/source-shopify-singer/source_shopify_singer/source.py index 4f4aad0a9aef0..abf0868753383 100644 --- a/airbyte-integrations/connectors/source-shopify-singer/source_shopify_singer/source.py +++ b/airbyte-integrations/connectors/source-shopify-singer/source_shopify_singer/source.py @@ -22,17 +22,23 @@ SOFTWARE. """ -import json +from typing import Dict import shopify -from airbyte_protocol import AirbyteCatalog, AirbyteConnectionStatus, Status +from airbyte_protocol import AirbyteCatalog from base_python import AirbyteLogger -from base_singer import SingerSource +from base_singer import BaseSingerSource -TAP_CMD = "tap-shopify" +class SourceShopifySinger(BaseSingerSource): + """ + Shopify API Reference: https://shopify.dev/docs/admin-api/rest/reference + """ + + tap_cmd = "tap-shopify" + tap_name = "Shopify API" + api_error = Exception -class SourceShopifySinger(SingerSource): def transform_config(self, raw_config): return { "start_date": raw_config["start_date"], @@ -41,24 +47,14 @@ def transform_config(self, raw_config): "date_window_size": 7, } - def check_config(self, logger: AirbyteLogger, config_path: str, config: json) -> AirbyteConnectionStatus: - try: - session = shopify.Session(f"{config['shop']}.myshopify.com", "2020-10", config["api_key"]) - shopify.ShopifyResource.activate_session(session) - # try to read the name of the shop, which should be available with any level of permissions - shopify.GraphQL().execute("{ shop { name id } }") - shopify.ShopifyResource.clear_session() - return AirbyteConnectionStatus(status=Status.SUCCEEDED) - except Exception as e: - logger.error(f"Exception connecting to Shopify: ${e}") - return AirbyteConnectionStatus( - status=Status.FAILED, message="Unable to connect to the Shopify API with the provided credentials." - ) - - def discover_cmd(self, logger: AirbyteLogger, config_path: str) -> str: - return f"{TAP_CMD} -c {config_path} --discover" + def try_connect(self, logger: AirbyteLogger, config: dict): + session = shopify.Session(f"{config['shop']}.myshopify.com", "2020-10", config["api_key"]) + shopify.ShopifyResource.activate_session(session) + # try to read the name of the shop, which should be available with any level of permissions + shopify.GraphQL().execute("{ shop { name id } }") + shopify.ShopifyResource.clear_session() - def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: + def discover(self, logger: AirbyteLogger, config: Dict[str, any]) -> AirbyteCatalog: catalog: AirbyteCatalog = super().discover(logger, config) # the metafields stream has a union type i.e: ["null", "integer", "object", "string"], but Airbyte doesn't currently allow the notion of a @@ -71,7 +67,3 @@ def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: schema["properties"]["value"]["type"] = ["null", "string"] return catalog - - def read_cmd(self, logger: AirbyteLogger, config_path: str, catalog_path: str, state_path: str = None) -> str: - state_path = f"--state {state_path}" if state_path else "" - return f"{TAP_CMD} -c {config_path} --catalog {catalog_path} {state_path}" From 1018c69c4a22e8d271ab115c6f044227ecafb3c1 Mon Sep 17 00:00:00 2001 From: vitaliizazmic <75620293+vitaliizazmic@users.noreply.github.com> Date: Thu, 28 Jan 2021 05:21:47 +0200 Subject: [PATCH 7/8] Looker source #1837 - CI tests failing (#1857) Co-authored-by: Sherif Nada --- .../00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json | 2 +- .../resources/seed/source_definitions.yaml | 2 +- .../connectors/source-looker/Dockerfile | 2 +- .../connectors/source-looker/setup.py | 2 +- .../source-looker/source_looker/client.py | 74 ++++++++++--------- 5 files changed, 42 insertions(+), 40 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json index ad5087f89e819..a189121bd4ba4 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c", "name": "Looker", "dockerRepository": "airbyte/source-looker", - "dockerImageTag": "0.1.0", + "dockerImageTag": "0.1.1", "documentationUrl": "https://hub.docker.com/r/airbyte/source-looker" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 52c6c0dd9b3db..904642a2a6692 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -161,7 +161,7 @@ - sourceDefinitionId: 00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c name: Looker dockerRepository: airbyte/source-looker - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 documentationUrl: https://hub.docker.com/r/airbyte/source-looker - sourceDefinitionId: ed799e2b-2158-4c66-8da4-b40fe63bc72a name: Plaid diff --git a/airbyte-integrations/connectors/source-looker/Dockerfile b/airbyte-integrations/connectors/source-looker/Dockerfile index 63093ac08293b..f52698cbf6f0e 100644 --- a/airbyte-integrations/connectors/source-looker/Dockerfile +++ b/airbyte-integrations/connectors/source-looker/Dockerfile @@ -12,5 +12,5 @@ COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ RUN pip install ".[main]" -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-looker diff --git a/airbyte-integrations/connectors/source-looker/setup.py b/airbyte-integrations/connectors/source-looker/setup.py index 77314e0e420d8..07993d2510c05 100644 --- a/airbyte-integrations/connectors/source-looker/setup.py +++ b/airbyte-integrations/connectors/source-looker/setup.py @@ -30,7 +30,7 @@ author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "base-python", "requests"], + install_requires=["airbyte-protocol", "base-python", "requests", "backoff"], package_data={"": ["*.json", "schemas/*.json"]}, setup_requires=["pytest-runner"], tests_require=["pytest"], diff --git a/airbyte-integrations/connectors/source-looker/source_looker/client.py b/airbyte-integrations/connectors/source-looker/source_looker/client.py index f9d063415ce83..c7f057bb5c0a8 100644 --- a/airbyte-integrations/connectors/source-looker/source_looker/client.py +++ b/airbyte-integrations/connectors/source-looker/source_looker/client.py @@ -24,6 +24,7 @@ from typing import List, Tuple +import backoff import requests from base_python import BaseClient from requests.exceptions import ConnectionError @@ -70,6 +71,7 @@ def health_check(self) -> Tuple[bool, str]: return False, self._connect_error return True, "" + @backoff.on_exception(backoff.expo, requests.exceptions.ConnectionError, max_tries=7) def _request(self, url: str, method: str = "GET", data: dict = None) -> List[dict]: response = requests.request(method, url, headers=self._headers, json=data) @@ -96,13 +98,13 @@ def _get_user_ids(self) -> List[int]: self._user_ids = [obj["id"] for obj in self._request(f"{self.BASE_URL}/users")] return self._user_ids - def stream__color_collections(self): + def stream__color_collections(self, fields): yield from self._request(f"{self.BASE_URL}/color_collections") - def stream__connections(self): + def stream__connections(self, fields): yield from self._request(f"{self.BASE_URL}/connections") - def stream__dashboards(self): + def stream__dashboards(self, fields): dashboards_list = [obj for obj in self._request(f"{self.BASE_URL}/dashboards") if isinstance(obj["id"], int)] self._dashboard_ids = [obj["id"] for obj in dashboards_list] self._context_metadata_mapping["dashboards"] = [ @@ -110,135 +112,135 @@ def stream__dashboards(self): ] yield from dashboards_list - def stream__dashboard_elements(self): + def stream__dashboard_elements(self, fields): for dashboard_id in self._get_dashboard_ids(): yield from self._request(f"{self.BASE_URL}/dashboards/{dashboard_id}/dashboard_elements") - def stream__dashboard_filters(self): + def stream__dashboard_filters(self, fields): for dashboard_id in self._get_dashboard_ids(): yield from self._request(f"{self.BASE_URL}/dashboards/{dashboard_id}/dashboard_filters") - def stream__dashboard_layouts(self): + def stream__dashboard_layouts(self, fields): for dashboard_id in self._get_dashboard_ids(): yield from self._request(f"{self.BASE_URL}/dashboards/{dashboard_id}/dashboard_layouts") - def stream__datagroups(self): + def stream__datagroups(self, fields): yield from self._request(f"{self.BASE_URL}/datagroups") - def stream__folders(self): + def stream__folders(self, fields): folders_list = self._request(f"{self.BASE_URL}/folders") self._context_metadata_mapping["folders"] = [ obj["content_metadata_id"] for obj in folders_list if isinstance(obj["content_metadata_id"], int) ] yield from folders_list - def stream__groups(self): + def stream__groups(self, fields): yield from self._request(f"{self.BASE_URL}/groups") - def stream__homepages(self): + def stream__homepages(self, fields): homepages_list = self._request(f"{self.BASE_URL}/homepages") self._context_metadata_mapping["homepages"] = [ obj["content_metadata_id"] for obj in homepages_list if isinstance(obj["content_metadata_id"], int) ] yield from homepages_list - def stream__integration_hubs(self): + def stream__integration_hubs(self, fields): yield from self._request(f"{self.BASE_URL}/integration_hubs") - def stream__integrations(self): + def stream__integrations(self, fields): yield from self._request(f"{self.BASE_URL}/integrations") - def stream__lookml_dashboards(self): + def stream__lookml_dashboards(self, fields): lookml_dashboards_list = [obj for obj in self._request(f"{self.BASE_URL}/dashboards") if isinstance(obj["id"], str)] yield from lookml_dashboards_list - def stream__lookml_models(self): + def stream__lookml_models(self, fields): yield from self._request(f"{self.BASE_URL}/lookml_models") - def stream__looks(self): + def stream__looks(self, fields): looks_list = self._request(f"{self.BASE_URL}/looks") self._context_metadata_mapping["looks"] = [ obj["content_metadata_id"] for obj in looks_list if isinstance(obj["content_metadata_id"], int) ] yield from looks_list - def stream__model_sets(self): + def stream__model_sets(self, fields): yield from self._request(f"{self.BASE_URL}/model_sets") - def stream__permission_sets(self): + def stream__permission_sets(self, fields): yield from self._request(f"{self.BASE_URL}/permission_sets") - def stream__permissions(self): + def stream__permissions(self, fields): yield from self._request(f"{self.BASE_URL}/permissions") - def stream__projects(self): + def stream__projects(self, fields): projects_list = self._request(f"{self.BASE_URL}/projects") self._project_ids = [obj["id"] for obj in projects_list] yield from projects_list - def stream__project_files(self): + def stream__project_files(self, fields): for project_id in self._get_project_ids(): yield from self._request(f"{self.BASE_URL}/projects/{project_id}/files") - def stream__git_branches(self): + def stream__git_branches(self, fields): for project_id in self._get_project_ids(): yield from self._request(f"{self.BASE_URL}/projects/{project_id}/git_branches") - def stream__roles(self): + def stream__roles(self, fields): roles_list = self._request(f"{self.BASE_URL}/roles") self._role_ids = [obj["id"] for obj in roles_list] yield from roles_list - def stream__role_groups(self): + def stream__role_groups(self, fields): if not self._role_ids: self._role_ids = [obj["id"] for obj in self._request(f"{self.BASE_URL}/roles")] for role_id in self._role_ids: yield from self._request(f"{self.BASE_URL}/roles/{role_id}/groups") - def stream__scheduled_plans(self): + def stream__scheduled_plans(self, fields): yield from self._request(f"{self.BASE_URL}/scheduled_plans?all_users=true") - def stream__spaces(self): + def stream__spaces(self, fields): spaces_list = self._request(f"{self.BASE_URL}/spaces") self._context_metadata_mapping["spaces"] = [ obj["content_metadata_id"] for obj in spaces_list if isinstance(obj["content_metadata_id"], int) ] yield from spaces_list - def stream__user_attributes(self): + def stream__user_attributes(self, fields): user_attributes_list = self._request(f"{self.BASE_URL}/user_attributes") self._user_attribute_ids = [obj["id"] for obj in user_attributes_list] yield from user_attributes_list - def stream__user_attribute_group_values(self): + def stream__user_attribute_group_values(self, fields): if not self._user_attribute_ids: self._user_attribute_ids = [obj["id"] for obj in self._request(f"{self.BASE_URL}/user_attributes")] for user_attribute_id in self._user_attribute_ids: yield from self._request(f"{self.BASE_URL}/user_attributes/{user_attribute_id}/group_values") - def stream__user_login_lockouts(self): + def stream__user_login_lockouts(self, fields): yield from self._request(f"{self.BASE_URL}/user_login_lockouts") - def stream__users(self): + def stream__users(self, fields): users_list = self._request(f"{self.BASE_URL}/users") self._user_ids = [obj["id"] for obj in users_list] yield from users_list - def stream__user_attribute_values(self): + def stream__user_attribute_values(self, fields): for user_ids in self._get_user_ids(): yield from self._request(f"{self.BASE_URL}/users/{user_ids}/attribute_values?all_values=true&include_unset=true") - def stream__user_sessions(self): + def stream__user_sessions(self, fields): for user_ids in self._get_user_ids(): yield from self._request(f"{self.BASE_URL}/users/{user_ids}/sessions") - def stream__versions(self): + def stream__versions(self, fields): yield from self._request(f"{self.BASE_URL}/versions") - def stream__workspaces(self): + def stream__workspaces(self, fields): yield from self._request(f"{self.BASE_URL}/workspaces") - def stream__query_history(self): + def stream__query_history(self, fields): request_data = { "model": "i__looker", "view": "history", @@ -263,10 +265,10 @@ def stream__query_history(self): for history_data in history_list: yield {k.replace(".", "_"): v for k, v in history_data.items()} - def stream__content_metadata(self): + def stream__content_metadata(self, fields): yield from self._metadata_processing(f"{self.BASE_URL}/content_metadata/") - def stream__content_metadata_access(self): + def stream__content_metadata_access(self, fields): yield from self._metadata_processing(f"{self.BASE_URL}/content_metadata_access?content_metadata_id=") def _metadata_processing(self, url: str): From 099c22eb2b99c05279308ccc5049c1bc46b6748b Mon Sep 17 00:00:00 2001 From: Artem Astapenko <3767150+Jamakase@users.noreply.github.com> Date: Thu, 28 Jan 2021 06:43:47 +0300 Subject: [PATCH 8/8] Hide job summary until attempt has succeeded or failed (#1863) --- .../components/JobItem/components/AttemptDetails.tsx | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx b/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx index 603fac3841b9c..e69f347600e5d 100644 --- a/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx +++ b/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx @@ -4,6 +4,7 @@ import styled from "styled-components"; import dayjs from "dayjs"; import { Attempt } from "../../../core/resources/Job"; +import Status from "../../../core/statuses"; type IProps = { className?: string; @@ -22,6 +23,17 @@ const AttemptDetails: React.FC = ({ className, configType }) => { + if (attempt.status !== Status.SUCCEEDED && attempt.status !== Status.FAILED) { + return ( +
+ +
+ ); + } + const formatBytes = (bytes: number) => { if (!bytes) { return (