From 4314a652e56071b1fd544dfa83041d84fb55f155 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 22 Sep 2023 00:03:15 +0200 Subject: [PATCH 01/12] Schema fixed --- .../source_amazon_ads/schemas/sponsored_brands.py | 5 +++++ .../source_amazon_ads/schemas/sponsored_products.py | 1 + 2 files changed, 6 insertions(+) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py index dba006e1b1562..b7c375fefaf7f 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py @@ -52,3 +52,8 @@ class BrandsAdGroup(CatalogModel): adGroupId: Decimal name: str bid: int + keywordId: Decimal + keywordText: str + nativeLanguageKeyword: str + matchType: str + state: str diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py index d60f1042592cd..b5ca604e06b69 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py @@ -32,6 +32,7 @@ class ProductCampaign(CatalogModel): endDate: str = None premiumBidAdjustment: bool bidding: Bidding + networks: str class ProductAdGroups(CatalogModel): From 58df40edc16e6488ac87ce80b4b6a3124a8d31e5 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 22 Sep 2023 00:06:07 +0200 Subject: [PATCH 02/12] Bump version --- airbyte-integrations/connectors/source-amazon-ads/Dockerfile | 2 +- airbyte-integrations/connectors/source-amazon-ads/metadata.yaml | 2 +- docs/integrations/sources/amazon-ads.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index 7a1e8e2d82abe..bbef6c16ee47e 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=3.2.0 +LABEL io.airbyte.version=3.2.1 LABEL io.airbyte.name=airbyte/source-amazon-ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml index 517408077b889..24987677c225e 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml @@ -8,7 +8,7 @@ data: connectorSubtype: api connectorType: source definitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 - dockerImageTag: 3.2.0 + dockerImageTag: 3.2.1 dockerRepository: airbyte/source-amazon-ads githubIssueLabel: source-amazon-ads icon: amazonads.svg diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index 25dc221cdb6b0..cc85696c85b83 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -109,6 +109,7 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 3.2.1 | 2023-09-22 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Fix unexpected column for SponsoredProductCampaigns and SponsoredBrandsKeywords | | 3.2.0 | 2023-09-18 | [30517](https://github.com/airbytehq/airbyte/pull/30517) | Add suggested streams; fix unexpected column issue | | 3.1.2 | 2023-08-16 | [29233](https://github.com/airbytehq/airbyte/pull/29233) | Add filter for Marketplace IDs | | 3.1.1 | 2023-08-28 | [29900](https://github.com/airbytehq/airbyte/pull/29900) | Add 404 handling for no assotiated with bid ad groups | From 65a82353a702f9c4cb0d4eeac929d2a23f360c35 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 22 Sep 2023 00:10:45 +0200 Subject: [PATCH 03/12] PR number updated --- docs/integrations/sources/amazon-ads.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index cc85696c85b83..073d3f718f839 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -109,7 +109,7 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| -| 3.2.1 | 2023-09-22 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Fix unexpected column for SponsoredProductCampaigns and SponsoredBrandsKeywords | +| 3.2.1 | 2023-09-22 | [30679](https://github.com/airbytehq/airbyte/pull/30679) | Fix unexpected column for SponsoredProductCampaigns and SponsoredBrandsKeywords | | 3.2.0 | 2023-09-18 | [30517](https://github.com/airbytehq/airbyte/pull/30517) | Add suggested streams; fix unexpected column issue | | 3.1.2 | 2023-08-16 | [29233](https://github.com/airbytehq/airbyte/pull/29233) | Add filter for Marketplace IDs | | 3.1.1 | 2023-08-28 | [29900](https://github.com/airbytehq/airbyte/pull/29900) | Add 404 handling for no assotiated with bid ad groups | From d2368e70aeae3154d37739e38c695bfcab7fb4c7 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 22 Sep 2023 00:41:30 +0200 Subject: [PATCH 04/12] Add format and patern to spec --- .../connectors/source-amazon-ads/acceptance-test-config.yml | 2 ++ .../connectors/source-amazon-ads/source_amazon_ads/spec.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml index d0be492d33693..5f27efd459338 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml @@ -61,5 +61,7 @@ acceptance_tests: spec: tests: - spec_path: integration_tests/spec.json + backward_compatibility_tests_config: + disable_for_version: "3.2.0" connector_image: airbyte/source-amazon-ads:dev test_strictness_level: high diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml index 0df1bdd6d002c..86b0f4fe27d0d 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml @@ -51,6 +51,8 @@ connectionSpecification: description: The Start date for collecting reports, should not be more than 60 days in the past. In YYYY-MM-DD format + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + format: date examples: - "2022-10-10" - "2022-10-22" From 2aed614b4b4d13150b44b28a48344f3aad578eca Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 22 Sep 2023 00:43:47 +0200 Subject: [PATCH 05/12] Set client id as airbyte_secret --- .../connectors/source-amazon-ads/source_amazon_ads/spec.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml index 86b0f4fe27d0d..09b4d52205f32 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml @@ -17,6 +17,7 @@ connectionSpecification: for more information. order: 1 type: string + airbyte_secret: true client_secret: title: Client Secret description: From 7bdc6a9a197192e038890e52f36f328777f19138 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 22 Sep 2023 00:55:06 +0200 Subject: [PATCH 06/12] Fix unit tests --- .../unit_tests/test_source.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py index 3dc310d11b250..d1e1a4f265c9f 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py @@ -3,6 +3,7 @@ # +import pytest import responses from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConnectorSpecification, Status, Type from jsonschema import Draft4Validator @@ -64,25 +65,23 @@ def test_check(config_gen): assert command_check(source, config_gen(start_date=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED) assert len(responses.calls) == 2 - assert command_check(source, config_gen(start_date="")) == AirbyteConnectionStatus(status=Status.SUCCEEDED) - assert len(responses.calls) == 4 + with pytest.raises(Exception): + command_check(source, config_gen(start_date="")) assert source.check(None, config_gen(start_date="2022-02-20")) == AirbyteConnectionStatus(status=Status.SUCCEEDED) - assert len(responses.calls) == 6 + assert len(responses.calls) == 4 assert command_check(source, config_gen(start_date="2022-20-02")) == AirbyteConnectionStatus( status=Status.FAILED, message="'month must be in 1..12'" ) - assert len(responses.calls) == 6 + assert len(responses.calls) == 4 - assert command_check(source, config_gen(start_date="no date")) == AirbyteConnectionStatus( - status=Status.FAILED, message="'String does not match format YYYY-MM-DD'" - ) - assert len(responses.calls) == 6 + with pytest.raises(Exception): + command_check(source, config_gen(start_date="no date")) assert command_check(source, config_gen(region=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED) - assert len(responses.calls) == 8 - assert url_strip_query(responses.calls[7].request.url) == "https://advertising-api.amazon.com/v2/profiles" + assert len(responses.calls) == 6 + assert url_strip_query(responses.calls[5].request.url) == "https://advertising-api.amazon.com/v2/profiles" assert command_check(source, config_gen(look_back_window=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED) From 22b0043aefc06341c8020694923efe96d4cbd269 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 22 Sep 2023 01:03:19 +0200 Subject: [PATCH 07/12] Deleted inapp doc --- docs/integrations/sources/amazon-ads.inapp.md | 20 ------------------- 1 file changed, 20 deletions(-) delete mode 100644 docs/integrations/sources/amazon-ads.inapp.md diff --git a/docs/integrations/sources/amazon-ads.inapp.md b/docs/integrations/sources/amazon-ads.inapp.md deleted file mode 100644 index d501c7be46423..0000000000000 --- a/docs/integrations/sources/amazon-ads.inapp.md +++ /dev/null @@ -1,20 +0,0 @@ -## Prerequisites - -- An [Amazon user](https://www.amazon.com) with access to an [Amazon Ads account](https://advertising.amazon.com) - -## Setup Guide - -1. Click `Authenticate your Amazon Ads account`. Log in and authorize access to the Amazon account. -2. Select **Region** to pull data from **North America (NA)**, **Europe (EU)**, **Far East (FE)**. See [Amazon Ads documentation](https://advertising.amazon.com/API/docs/en-us/info/api-overview#api-endpoints) for more details. -3. (Optional) **Start Date** can be used to generate reports starting from the specified start date in the format YYYY-MM-DD. The date should not be more than 60 days in the past. If not specified, today's date is used. The date is treated in the timezone of the processed profile. -4. (Optional) **Profile ID(s)** you want to fetch data for. A profile is an advertiser's account in a specific marketplace. See [Amazon Ads docs](https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles) for more details. If not specified, data from all Profiles will be synced. -5. (Optional) **State Filter** Filter for Display, Product, and Brand Campaign streams with a state of enabled, paused, or archived. If not specified, all streams regardless of state will be synced. -6. (Optional) **Look Back Window** The amount of days to go back in time to get the updated data from Amazon Ads. After the first sync, data from this date will be synced. -7. (Optional) **Report Record Types** Optional configuration which accepts an array of string of record types. Leave blank for default behaviour to pull all report types. Use this config option only if you want to pull specific report type(s). See [Amazon Ads docs](https://advertising.amazon.com/API/docs/en-us/reporting/v2/report-types) for more details -9. Click `Set up source`. - -### Report Timezones - -All the reports are generated relative to the target profile' timezone. - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Amazon Ads](https://docs.airbyte.com/integrations/sources/amazon-ads). From 16070afae6e83413a7372657a7b4e9a07e11f0e7 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 22 Sep 2023 01:06:35 +0200 Subject: [PATCH 08/12] Added format, pattern and airbyte_secret to spec.json --- .../connectors/source-amazon-ads/integration_tests/spec.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json index 715c567b4ce50..585149544f872 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json @@ -13,6 +13,7 @@ "client_id": { "title": "Client ID", "description": "The client ID of your Amazon Ads developer application. See the docs for more information.", + "airbyte_secret": true, "order": 1, "type": "string" }, @@ -43,7 +44,9 @@ "description": "The Start date for collecting reports, should not be more than 60 days in the past. In YYYY-MM-DD format", "examples": ["2022-10-10", "2022-10-22"], "order": 5, - "type": "string" + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "format": "date" }, "profiles": { "title": "Profile IDs", From dcc7771cfd1551e1acc53964c58149757e25d607 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 22 Sep 2023 12:58:52 +0200 Subject: [PATCH 09/12] Bumped version --- airbyte-integrations/connectors/source-amazon-ads/Dockerfile | 2 +- airbyte-integrations/connectors/source-amazon-ads/metadata.yaml | 2 +- docs/integrations/sources/amazon-ads.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index bbef6c16ee47e..7dc8eb5652368 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=3.2.1 +LABEL io.airbyte.version=3.3.0 LABEL io.airbyte.name=airbyte/source-amazon-ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml index 24987677c225e..458d14f52f01b 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml @@ -8,7 +8,7 @@ data: connectorSubtype: api connectorType: source definitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 - dockerImageTag: 3.2.1 + dockerImageTag: 3.3.0 dockerRepository: airbyte/source-amazon-ads githubIssueLabel: source-amazon-ads icon: amazonads.svg diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index 073d3f718f839..680307eac4992 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -109,7 +109,7 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| -| 3.2.1 | 2023-09-22 | [30679](https://github.com/airbytehq/airbyte/pull/30679) | Fix unexpected column for SponsoredProductCampaigns and SponsoredBrandsKeywords | +| 3.3.0 | 2023-09-22 | [30679](https://github.com/airbytehq/airbyte/pull/30679) | Fix unexpected column for `SponsoredProductCampaigns` and `SponsoredBrandsKeywords` | | 3.2.0 | 2023-09-18 | [30517](https://github.com/airbytehq/airbyte/pull/30517) | Add suggested streams; fix unexpected column issue | | 3.1.2 | 2023-08-16 | [29233](https://github.com/airbytehq/airbyte/pull/29233) | Add filter for Marketplace IDs | | 3.1.1 | 2023-08-28 | [29900](https://github.com/airbytehq/airbyte/pull/29900) | Add 404 handling for no assotiated with bid ad groups | From ad2de99ff282d3f9ae67598549ed822b9a12fb73 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Fri, 22 Sep 2023 11:03:53 +0000 Subject: [PATCH 10/12] Automated Commit - Formatting Changes --- .../destination-duckdb/devcontainer.json | 30 +- airbyte-cdk/python/airbyte_cdk/entrypoint.py | 2 +- .../models/declarative_component_schema.py | 1016 +++++++------- .../BigQueryDenormalizedDestination.java | 4 - .../bigquery/BigQueryAsyncFlush.java | 1 - .../bigquery/BigQueryDestination.java | 2 +- .../bigquery/BigQueryGcsOperations.java | 46 +- .../BigQueryStagingConsumerFactory.java | 12 +- .../bigquery/BigQueryWriteConfig.java | 1 + .../BigQueryDestinationHandler.java | 3 +- .../BigQueryGcsDestinationAcceptanceTest.java | 1 - .../destination_duckdb/destination.py | 28 +- .../SnowflakeDestinationHandler.java | 3 +- .../SnowflakeSqlGenerator.java | 34 +- .../SnowflakeSqlGeneratorIntegrationTest.java | 1 - .../SnowflakeSqlGeneratorTest.java | 31 +- .../source-apify-dataset/metadata.yaml | 2 +- .../integration_tests/configured_catalog.json | 132 +- .../source_bigcommerce/schemas/channels.json | 2 +- .../source_bigcommerce/schemas/pages.json | 2 +- .../schemas/campaigns.json | 2 +- .../integration_tests/configured_catalog.json | 190 ++- .../integration_tests/state.json | 2 +- .../integration_tests/abnormal_state.json | 2 +- .../integration_tests/configured_catalog.json | 6 +- .../defaults/default_reports.json | 336 ++--- .../integration_tests/configured_catalog.json | 2 +- .../source_jira/schemas/filter_sharing.json | 4 +- .../source_jira/schemas/issue_comments.json | 2 +- .../schemas/issue_custom_field_contexts.json | 4 +- .../source_jira/schemas/issue_properties.json | 2 +- .../schemas/issue_remote_links.json | 2 +- .../source_jira/schemas/issue_votes.json | 2 +- .../source_jira/schemas/issue_watchers.json | 2 +- .../source_jira/schemas/project_avatars.json | 4 +- .../schemas/project_permission_schemes.json | 4 +- .../schemas/screen_tab_fields.json | 8 +- .../source_jira/schemas/screen_tabs.json | 4 +- .../source_jira/type_transfromer.py | 4 + .../unit_tests/test_date_time_transformer.py | 4 + .../test_migrations/test_config.json | 11 +- .../mssql/MssqlCdcSavedInfoFetcher.java | 3 +- ...StrictEncryptJdbcSourceAcceptanceTest.java | 86 +- .../source/mysql/MySqlCdcProperties.java | 9 +- .../source/mysql/MySqlCdcStateHandler.java | 1 + .../source/mysql/MySqlQueryUtils.java | 1 - .../MySqlCursorBasedStateManager.java | 5 + .../sources/CdcMySqlSourceAcceptanceTest.java | 16 +- ...lSslCaCertificateSourceAcceptanceTest.java | 1 + ...cMySqlSslRequiredSourceAcceptanceTest.java | 1 + .../sources/MySqlDatatypeAccuracyTest.java | 1 + .../sources/MySqlSourceDatatypeTest.java | 1 + .../source/mysql/CdcMysqlSourceTest.java | 11 +- .../mysql/MySqlJdbcSourceAcceptanceTest.java | 196 ++- .../integration_tests/abnormal_state.json | 2 +- .../source_paypal_transaction/manifest.yaml | 30 +- .../integration_tests/configured_catalog.json | 2 +- .../source_pipedrive/manifest.yaml | 14 +- .../schemas/activity_fields.json | 2 +- .../source_pipedrive/schemas/deal_fields.json | 2 +- .../schemas/organization_fields.json | 10 +- .../schemas/organizations.json | 4 +- .../source/postgres/PostgresQueryUtils.java | 8 +- .../source/postgres/PostgresSource.java | 8 +- .../cdc/PostgresCdcCtidInitializer.java | 4 +- .../cdc/PostgresCdcSavedInfoFetcher.java | 2 +- .../source/postgres/ctid/Ctid.java | 1 + .../source/postgres/ctid/CtidUtils.java | 6 +- .../ctid/InitialSyncCtidIterator.java | 1 + ...actCdcPostgresSourceSslAcceptanceTest.java | 1 + ...ertificateSslAcceptanceLegacyCtidTest.java | 8 +- ...sSourceCaCertificateSslAcceptanceTest.java | 1 - ...ourceFullCertificateSslAcceptanceTest.java | 1 + ...ostgresSourceAcceptanceLegacyCtidTest.java | 8 +- .../CdcPostgresSourceAcceptanceTest.java | 1 + ...ostgresSourceAcceptanceLegacyCtidTest.java | 7 +- .../CdcPostgresSourceLegacyCtidTest.java | 5 + .../postgres/CdcPostgresSourceTest.java | 16 +- .../ctid/InitialSyncCtidIteratorTest.java | 64 +- .../source_sentry/schemas/events.json | 190 +-- .../source_sentry/schemas/issues.json | 10 +- .../source_sentry/schemas/project_detail.json | 60 +- .../source_sentry/schemas/projects.json | 18 +- .../source_sentry/schemas/releases.json | 38 +- .../integration_tests/abnormal_state.json | 1196 ++++++++--------- .../integration_tests/state.json | 488 +++---- .../source_slack/schemas/threads.json | 2 +- .../schemas/shared/tax_rate.json | 82 +- .../source-trello/source_trello/manifest.yaml | 45 +- .../schemas/deleted_tickets.json | 14 +- 90 files changed, 2281 insertions(+), 2352 deletions(-) diff --git a/.devcontainer/destination-duckdb/devcontainer.json b/.devcontainer/destination-duckdb/devcontainer.json index 587b51126ff68..cd04dcafb77d4 100644 --- a/.devcontainer/destination-duckdb/devcontainer.json +++ b/.devcontainer/destination-duckdb/devcontainer.json @@ -23,21 +23,21 @@ "customizations": { "vscode": { "extensions": [ - // Python extensions: - "charliermarsh.ruff", - "matangover.mypy", - "ms-python.black-formatter", - "ms-python.python", - "ms-python.vscode-pylance", + // Python extensions: + "charliermarsh.ruff", + "matangover.mypy", + "ms-python.black-formatter", + "ms-python.python", + "ms-python.vscode-pylance", - // Toml support - "tamasfe.even-better-toml", - - // Yaml and JSON Schema support: - "redhat.vscode-yaml", + // Toml support + "tamasfe.even-better-toml", - // Contributing: - "GitHub.vscode-pull-request-github" + // Yaml and JSON Schema support: + "redhat.vscode-yaml", + + // Contributing: + "GitHub.vscode-pull-request-github" ], "settings": { "extensions.ignoreRecommendations": true, @@ -53,8 +53,8 @@ "." ], "[python]": { - "editor.defaultFormatter": "ms-python.black-formatter" - } + "editor.defaultFormatter": "ms-python.black-formatter" + } } } }, diff --git a/airbyte-cdk/python/airbyte_cdk/entrypoint.py b/airbyte-cdk/python/airbyte_cdk/entrypoint.py index dd5e5724d1582..f29039835788d 100644 --- a/airbyte-cdk/python/airbyte_cdk/entrypoint.py +++ b/airbyte-cdk/python/airbyte_cdk/entrypoint.py @@ -7,7 +7,6 @@ import ipaddress import logging import os.path -import requests import socket import sys import tempfile @@ -15,6 +14,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union from urllib.parse import urlparse +import requests from airbyte_cdk.connector import TConfig from airbyte_cdk.exception_handler import init_uncaught_exception_handler from airbyte_cdk.logger import init_logger diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index c790291e9fc05..24e3a907b1151 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -11,12 +11,12 @@ class AddedFieldDefinition(BaseModel): - type: Literal['AddedFieldDefinition'] + type: Literal["AddedFieldDefinition"] path: List[str] = Field( ..., - description='List of strings defining the path where to add the value on the record.', - examples=[['segment_id'], ['metadata', 'segment_id']], - title='Path', + description="List of strings defining the path where to add the value on the record.", + examples=[["segment_id"], ["metadata", "segment_id"]], + title="Path", ) value: str = Field( ..., @@ -26,607 +26,601 @@ class AddedFieldDefinition(BaseModel): "{{ record['MetaData']['LastUpdatedTime'] }}", "{{ stream_partition['segment_id'] }}", ], - title='Value', + title="Value", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class AddFields(BaseModel): - type: Literal['AddFields'] + type: Literal["AddFields"] fields: List[AddedFieldDefinition] = Field( ..., - description='List of transformations (path and corresponding value) that will be added to the record.', - title='Fields', + description="List of transformations (path and corresponding value) that will be added to the record.", + title="Fields", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class AuthFlowType(Enum): - oauth2_0 = 'oauth2.0' - oauth1_0 = 'oauth1.0' + oauth2_0 = "oauth2.0" + oauth1_0 = "oauth1.0" class BasicHttpAuthenticator(BaseModel): - type: Literal['BasicHttpAuthenticator'] + type: Literal["BasicHttpAuthenticator"] username: str = Field( ..., - description='The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.', + description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], - title='Username', + title="Username", ) password: Optional[str] = Field( - '', - description='The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.', - examples=["{{ config['password'] }}", ''], - title='Password', + "", + description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", + examples=["{{ config['password'] }}", ""], + title="Password", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class BearerAuthenticator(BaseModel): - type: Literal['BearerAuthenticator'] + type: Literal["BearerAuthenticator"] api_token: str = Field( ..., - description='Token to inject as request header for authenticating with the API.', + description="Token to inject as request header for authenticating with the API.", examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], - title='Bearer Token', + title="Bearer Token", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CheckStream(BaseModel): - type: Literal['CheckStream'] + type: Literal["CheckStream"] stream_names: List[str] = Field( ..., - description='Names of the streams to try reading from when running a check operation.', - examples=[['users'], ['users', 'contacts']], - title='Stream Names', + description="Names of the streams to try reading from when running a check operation.", + examples=[["users"], ["users", "contacts"]], + title="Stream Names", ) class ConstantBackoffStrategy(BaseModel): - type: Literal['ConstantBackoffStrategy'] + type: Literal["ConstantBackoffStrategy"] backoff_time_in_seconds: Union[float, str] = Field( ..., - description='Backoff time in seconds.', + description="Backoff time in seconds.", examples=[30, 30.5, "{{ config['backoff_time'] }}"], - title='Backoff Time', + title="Backoff Time", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomAuthenticator(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomAuthenticator'] + type: Literal["CustomAuthenticator"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_..`.', - examples=['source_railz.components.ShortLivedTokenAuthenticator'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_..`.", + examples=["source_railz.components.ShortLivedTokenAuthenticator"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomBackoffStrategy(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomBackoffStrategy'] + type: Literal["CustomBackoffStrategy"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomBackoffStrategy'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomBackoffStrategy"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomErrorHandler(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomErrorHandler'] + type: Literal["CustomErrorHandler"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_..`.', - examples=['source_railz.components.MyCustomErrorHandler'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_..`.", + examples=["source_railz.components.MyCustomErrorHandler"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomIncrementalSync(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomIncrementalSync'] + type: Literal["CustomIncrementalSync"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_..`.', - examples=['source_railz.components.MyCustomIncrementalSync'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_..`.", + examples=["source_railz.components.MyCustomIncrementalSync"], + title="Class Name", ) cursor_field: str = Field( ..., - description='The location of the value on a record that will be used as a bookmark during sync.', + description="The location of the value on a record that will be used as a bookmark during sync.", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomPaginationStrategy(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomPaginationStrategy'] + type: Literal["CustomPaginationStrategy"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomPaginationStrategy'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomPaginationStrategy"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomRecordExtractor(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomRecordExtractor'] + type: Literal["CustomRecordExtractor"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomRecordExtractor'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomRecordExtractor"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomRequester(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomRequester'] + type: Literal["CustomRequester"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomRecordExtractor'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomRecordExtractor"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomRetriever(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomRetriever'] + type: Literal["CustomRetriever"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomRetriever'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomRetriever"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomPartitionRouter(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomPartitionRouter'] + type: Literal["CustomPartitionRouter"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_..`.', - examples=['source_railz.components.MyCustomPartitionRouter'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_..`.", + examples=["source_railz.components.MyCustomPartitionRouter"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomTransformation(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomTransformation'] + type: Literal["CustomTransformation"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_..`.', - examples=['source_railz.components.MyCustomTransformation'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_..`.", + examples=["source_railz.components.MyCustomTransformation"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class RefreshTokenUpdater(BaseModel): refresh_token_name: Optional[str] = Field( - 'refresh_token', - description='The name of the property which contains the updated refresh token in the response from the token refresh endpoint.', - examples=['refresh_token'], - title='Refresh Token Property Name', + "refresh_token", + description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", + examples=["refresh_token"], + title="Refresh Token Property Name", ) access_token_config_path: Optional[List[str]] = Field( - ['credentials', 'access_token'], - description='Config path to the access token. Make sure the field actually exists in the config.', - examples=[['credentials', 'access_token'], ['access_token']], - title='Config Path To Access Token', + ["credentials", "access_token"], + description="Config path to the access token. Make sure the field actually exists in the config.", + examples=[["credentials", "access_token"], ["access_token"]], + title="Config Path To Access Token", ) refresh_token_config_path: Optional[List[str]] = Field( - ['credentials', 'refresh_token'], - description='Config path to the access token. Make sure the field actually exists in the config.', - examples=[['credentials', 'refresh_token'], ['refresh_token']], - title='Config Path To Refresh Token', + ["credentials", "refresh_token"], + description="Config path to the access token. Make sure the field actually exists in the config.", + examples=[["credentials", "refresh_token"], ["refresh_token"]], + title="Config Path To Refresh Token", ) token_expiry_date_config_path: Optional[List[str]] = Field( - ['credentials', 'token_expiry_date'], - description='Config path to the expiry date. Make sure actually exists in the config.', - examples=[['credentials', 'token_expiry_date']], - title='Config Path To Expiry Date', + ["credentials", "token_expiry_date"], + description="Config path to the expiry date. Make sure actually exists in the config.", + examples=[["credentials", "token_expiry_date"]], + title="Config Path To Expiry Date", ) class OAuthAuthenticator(BaseModel): - type: Literal['OAuthAuthenticator'] + type: Literal["OAuthAuthenticator"] client_id: str = Field( ..., - description='The OAuth client ID. Fill it in the user inputs.', + description="The OAuth client ID. Fill it in the user inputs.", examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], - title='Client ID', + title="Client ID", ) client_secret: str = Field( ..., - description='The OAuth client secret. Fill it in the user inputs.', + description="The OAuth client secret. Fill it in the user inputs.", examples=[ "{{ config['client_secret }}", "{{ config['credentials']['client_secret }}", ], - title='Client Secret', + title="Client Secret", ) refresh_token: Optional[str] = Field( None, - description='Credential artifact used to get a new access token.', + description="Credential artifact used to get a new access token.", examples=[ "{{ config['refresh_token'] }}", "{{ config['credentials]['refresh_token'] }}", ], - title='Refresh Token', + title="Refresh Token", ) token_refresh_endpoint: str = Field( ..., - description='The full URL to call to obtain a new access token.', - examples=['https://connect.squareup.com/oauth2/token'], - title='Token Refresh Endpoint', + description="The full URL to call to obtain a new access token.", + examples=["https://connect.squareup.com/oauth2/token"], + title="Token Refresh Endpoint", ) access_token_name: Optional[str] = Field( - 'access_token', - description='The name of the property which contains the access token in the response from the token refresh endpoint.', - examples=['access_token'], - title='Access Token Property Name', + "access_token", + description="The name of the property which contains the access token in the response from the token refresh endpoint.", + examples=["access_token"], + title="Access Token Property Name", ) expires_in_name: Optional[str] = Field( - 'expires_in', - description='The name of the property which contains the expiry date in the response from the token refresh endpoint.', - examples=['expires_in'], - title='Token Expiry Property Name', + "expires_in", + description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", + examples=["expires_in"], + title="Token Expiry Property Name", ) grant_type: Optional[str] = Field( - 'refresh_token', - description='Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.', - examples=['refresh_token', 'client_credentials'], - title='Grant Type', + "refresh_token", + description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", + examples=["refresh_token", "client_credentials"], + title="Grant Type", ) refresh_request_body: Optional[Dict[str, Any]] = Field( None, - description='Body of the request sent to get a new access token.', + description="Body of the request sent to get a new access token.", examples=[ { - 'applicationId': "{{ config['application_id'] }}", - 'applicationSecret': "{{ config['application_secret'] }}", - 'token': "{{ config['token'] }}", + "applicationId": "{{ config['application_id'] }}", + "applicationSecret": "{{ config['application_secret'] }}", + "token": "{{ config['token'] }}", } ], - title='Refresh Request Body', + title="Refresh Request Body", ) scopes: Optional[List[str]] = Field( None, - description='List of scopes that should be granted to the access token.', - examples=[ - ['crm.list.read', 'crm.objects.contacts.read', 'crm.schema.contacts.read'] - ], - title='Scopes', + description="List of scopes that should be granted to the access token.", + examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], + title="Scopes", ) token_expiry_date: Optional[str] = Field( None, - description='The access token expiry date.', - examples=['2023-04-06T07:12:10.421833+00:00', 1680842386], - title='Token Expiry Date', + description="The access token expiry date.", + examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], + title="Token Expiry Date", ) token_expiry_date_format: Optional[str] = Field( None, - description='The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.', - examples=['%Y-%m-%d %H:%M:%S.%f+00:00'], - title='Token Expiry Date Format', + description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", + examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], + title="Token Expiry Date Format", ) refresh_token_updater: Optional[RefreshTokenUpdater] = Field( None, - description='When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.', - title='Token Updater', + description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", + title="Token Updater", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class ExponentialBackoffStrategy(BaseModel): - type: Literal['ExponentialBackoffStrategy'] + type: Literal["ExponentialBackoffStrategy"] factor: Optional[Union[float, str]] = Field( 5, - description='Multiplicative constant applied on each retry.', - examples=[5, 5.5, '10'], - title='Factor', + description="Multiplicative constant applied on each retry.", + examples=[5, 5.5, "10"], + title="Factor", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SessionTokenRequestBearerAuthenticator(BaseModel): - type: Literal['Bearer'] + type: Literal["Bearer"] class HttpMethodEnum(Enum): - GET = 'GET' - POST = 'POST' + GET = "GET" + POST = "POST" class Action(Enum): - SUCCESS = 'SUCCESS' - FAIL = 'FAIL' - RETRY = 'RETRY' - IGNORE = 'IGNORE' + SUCCESS = "SUCCESS" + FAIL = "FAIL" + RETRY = "RETRY" + IGNORE = "IGNORE" class HttpResponseFilter(BaseModel): - type: Literal['HttpResponseFilter'] + type: Literal["HttpResponseFilter"] action: Action = Field( ..., - description='Action to execute if a response matches the filter.', - examples=['SUCCESS', 'FAIL', 'RETRY', 'IGNORE'], - title='Action', + description="Action to execute if a response matches the filter.", + examples=["SUCCESS", "FAIL", "RETRY", "IGNORE"], + title="Action", ) error_message: Optional[str] = Field( None, - description='Error Message to display if the response matches the filter.', - title='Error Message', + description="Error Message to display if the response matches the filter.", + title="Error Message", ) error_message_contains: Optional[str] = Field( None, - description='Match the response if its error message contains the substring.', - example=['This API operation is not enabled for this site'], - title='Error Message Substring', + description="Match the response if its error message contains the substring.", + example=["This API operation is not enabled for this site"], + title="Error Message Substring", ) http_codes: Optional[List[int]] = Field( None, - description='Match the response if its HTTP code is included in this list.', + description="Match the response if its HTTP code is included in this list.", examples=[[420, 429], [500]], - title='HTTP Codes', + title="HTTP Codes", ) predicate: Optional[str] = Field( None, - description='Match the response if the predicate evaluates to true.', + description="Match the response if the predicate evaluates to true.", examples=[ "{{ 'Too much requests' in response }}", "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", ], - title='Predicate', + title="Predicate", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class InlineSchemaLoader(BaseModel): - type: Literal['InlineSchemaLoader'] + type: Literal["InlineSchemaLoader"] schema_: Optional[Dict[str, Any]] = Field( None, - alias='schema', + alias="schema", description='Describes a streams\' schema. Refer to the Data Types documentation for more details on which types are valid.', - title='Schema', + title="Schema", ) class JsonFileSchemaLoader(BaseModel): - type: Literal['JsonFileSchemaLoader'] + type: Literal["JsonFileSchemaLoader"] file_path: Optional[str] = Field( None, description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", - example=['./schemas/users.json'], - title='File Path', + example=["./schemas/users.json"], + title="File Path", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class JsonDecoder(BaseModel): - type: Literal['JsonDecoder'] + type: Literal["JsonDecoder"] class MinMaxDatetime(BaseModel): - type: Literal['MinMaxDatetime'] + type: Literal["MinMaxDatetime"] datetime: str = Field( ..., - description='Datetime value.', - examples=['2021-01-01', '2021-01-01T00:00:00Z', "{{ config['start_time'] }}"], - title='Datetime', + description="Datetime value.", + examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], + title="Datetime", ) datetime_format: Optional[str] = Field( - '', + "", description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', - examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s'], - title='Datetime Format', + examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], + title="Datetime Format", ) max_datetime: Optional[str] = Field( None, - description='Ceiling applied on the datetime value. Must be formatted with the datetime_format field.', - examples=['2021-01-01T00:00:00Z', '2021-01-01'], - title='Max Datetime', + description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", + examples=["2021-01-01T00:00:00Z", "2021-01-01"], + title="Max Datetime", ) min_datetime: Optional[str] = Field( None, - description='Floor applied on the datetime value. Must be formatted with the datetime_format field.', - examples=['2010-01-01T00:00:00Z', '2010-01-01'], - title='Min Datetime', + description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", + examples=["2010-01-01T00:00:00Z", "2010-01-01"], + title="Min Datetime", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class NoAuth(BaseModel): - type: Literal['NoAuth'] - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + type: Literal["NoAuth"] + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class NoPagination(BaseModel): - type: Literal['NoPagination'] + type: Literal["NoPagination"] class OAuthConfigSpecification(BaseModel): class Config: extra = Extra.allow - oauth_user_input_from_connector_config_specification: Optional[ - Dict[str, Any] - ] = Field( + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", examples=[ - {'app_id': {'type': 'string', 'path_in_connector_config': ['app_id']}}, + {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, { - 'app_id': { - 'type': 'string', - 'path_in_connector_config': ['info', 'app_id'], + "app_id": { + "type": "string", + "path_in_connector_config": ["info", "app_id"], } }, ], - title='OAuth user input', + title="OAuth user input", ) complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", examples=[ { - 'refresh_token': { - 'type': 'string,', - 'path_in_connector_config': ['credentials', 'refresh_token'], + "refresh_token": { + "type": "string,", + "path_in_connector_config": ["credentials", "refresh_token"], } } ], - title='OAuth output specification', + title="OAuth output specification", ) complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( None, - description='OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }', - examples=[ - {'client_id': {'type': 'string'}, 'client_secret': {'type': 'string'}} - ], - title='OAuth input specification', + description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", + examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], + title="OAuth input specification", ) complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", examples=[ { - 'client_id': { - 'type': 'string,', - 'path_in_connector_config': ['credentials', 'client_id'], + "client_id": { + "type": "string,", + "path_in_connector_config": ["credentials", "client_id"], }, - 'client_secret': { - 'type': 'string,', - 'path_in_connector_config': ['credentials', 'client_secret'], + "client_secret": { + "type": "string,", + "path_in_connector_config": ["credentials", "client_secret"], }, } ], - title='OAuth server output specification', + title="OAuth server output specification", ) class OffsetIncrement(BaseModel): - type: Literal['OffsetIncrement'] + type: Literal["OffsetIncrement"] page_size: Optional[Union[int, str]] = Field( None, - description='The number of records to include in each pages.', + description="The number of records to include in each pages.", examples=[100, "{{ config['page_size'] }}"], - title='Limit', + title="Limit", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class PageIncrement(BaseModel): - type: Literal['PageIncrement'] + type: Literal["PageIncrement"] page_size: Optional[int] = Field( None, - description='The number of records to include in each pages.', - examples=[100, '100'], - title='Page Size', + description="The number of records to include in each pages.", + examples=[100, "100"], + title="Page Size", ) start_from_page: Optional[int] = Field( 0, - description='Index of the first page to request.', + description="Index of the first page to request.", examples=[0, 1], - title='Start From Page', + title="Start From Page", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class PrimaryKey(BaseModel): __root__: Union[str, List[str], List[List[str]]] = Field( ..., - description='The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.', - examples=['id', ['code', 'type']], - title='Primary Key', + description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", + examples=["id", ["code", "type"]], + title="Primary Key", ) class RecordFilter(BaseModel): - type: Literal['RecordFilter'] + type: Literal["RecordFilter"] condition: Optional[str] = Field( - '', - description='The predicate to filter a record. Records will be removed if evaluated to False.', + "", + description="The predicate to filter a record. Records will be removed if evaluated to False.", examples=[ "{{ record['created_at'] >= stream_interval['start_time'] }}", "{{ record.status in ['active', 'expired'] }}", ], ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class RemoveFields(BaseModel): - type: Literal['RemoveFields'] + type: Literal["RemoveFields"] field_pointers: List[List[str]] = Field( ..., - description='Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.', - examples=[['tags'], [['content', 'html'], ['content', 'plain_text']]], - title='Field Paths', + description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", + examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], + title="Field Paths", ) class RequestPath(BaseModel): - type: Literal['RequestPath'] + type: Literal["RequestPath"] class InjectInto(Enum): - request_parameter = 'request_parameter' - header = 'header' - body_data = 'body_data' - body_json = 'body_json' + request_parameter = "request_parameter" + header = "header" + body_data = "body_data" + body_json = "body_json" class RequestOption(BaseModel): - type: Literal['RequestOption'] + type: Literal["RequestOption"] field_name: str = Field( ..., - description='Configures which key should be used in the location that the descriptor is being injected into', - examples=['segment_id'], - title='Request Option', + description="Configures which key should be used in the location that the descriptor is being injected into", + examples=["segment_id"], + title="Request Option", ) inject_into: InjectInto = Field( ..., - description='Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.', - examples=['request_parameter', 'header', 'body_data', 'body_json'], - title='Inject Into', + description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", + examples=["request_parameter", "header", "body_data", "body_json"], + title="Inject Into", ) @@ -638,253 +632,251 @@ class Config: class LegacySessionTokenAuthenticator(BaseModel): - type: Literal['LegacySessionTokenAuthenticator'] + type: Literal["LegacySessionTokenAuthenticator"] header: str = Field( ..., - description='The name of the session token header that will be injected in the request', - examples=['X-Session'], - title='Session Request Header', + description="The name of the session token header that will be injected in the request", + examples=["X-Session"], + title="Session Request Header", ) login_url: str = Field( ..., - description='Path of the login URL (do not include the base URL)', - examples=['session'], - title='Login Path', + description="Path of the login URL (do not include the base URL)", + examples=["session"], + title="Login Path", ) session_token: Optional[str] = Field( None, - description='Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair', + description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", example=["{{ config['session_token'] }}"], - title='Session Token', + title="Session Token", ) session_token_response_key: str = Field( ..., - description='Name of the key of the session token to be extracted from the response', - examples=['id'], - title='Response Token Response Key', + description="Name of the key of the session token to be extracted from the response", + examples=["id"], + title="Response Token Response Key", ) username: Optional[str] = Field( None, - description='Username used to authenticate and obtain a session token', + description="Username used to authenticate and obtain a session token", examples=[" {{ config['username'] }}"], - title='Username', + title="Username", ) password: Optional[str] = Field( - '', - description='Password used to authenticate and obtain a session token', - examples=["{{ config['password'] }}", ''], - title='Password', + "", + description="Password used to authenticate and obtain a session token", + examples=["{{ config['password'] }}", ""], + title="Password", ) validate_session_url: str = Field( ..., - description='Path of the URL to use to validate that the session token is valid (do not include the base URL)', - examples=['user/current'], - title='Validate Session Path', + description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", + examples=["user/current"], + title="Validate Session Path", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class WaitTimeFromHeader(BaseModel): - type: Literal['WaitTimeFromHeader'] + type: Literal["WaitTimeFromHeader"] header: str = Field( ..., - description='The name of the response header defining how long to wait before retrying.', - examples=['Retry-After'], - title='Response Header Name', + description="The name of the response header defining how long to wait before retrying.", + examples=["Retry-After"], + title="Response Header Name", ) regex: Optional[str] = Field( None, - description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.', - examples=['([-+]?\\d+)'], - title='Extraction Regex', + description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", + examples=["([-+]?\\d+)"], + title="Extraction Regex", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class WaitUntilTimeFromHeader(BaseModel): - type: Literal['WaitUntilTimeFromHeader'] + type: Literal["WaitUntilTimeFromHeader"] header: str = Field( ..., - description='The name of the response header defining how long to wait before retrying.', - examples=['wait_time'], - title='Response Header', + description="The name of the response header defining how long to wait before retrying.", + examples=["wait_time"], + title="Response Header", ) min_wait: Optional[Union[float, str]] = Field( None, - description='Minimum time to wait before retrying.', - examples=[10, '60'], - title='Minimum Wait Time', + description="Minimum time to wait before retrying.", + examples=[10, "60"], + title="Minimum Wait Time", ) regex: Optional[str] = Field( None, - description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.', - examples=['([-+]?\\d+)'], - title='Extraction Regex', + description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", + examples=["([-+]?\\d+)"], + title="Extraction Regex", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class ApiKeyAuthenticator(BaseModel): - type: Literal['ApiKeyAuthenticator'] + type: Literal["ApiKeyAuthenticator"] api_token: Optional[str] = Field( None, - description='The API key to inject in the request. Fill it in the user inputs.', + description="The API key to inject in the request. Fill it in the user inputs.", examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], - title='API Key', + title="API Key", ) header: Optional[str] = Field( None, - description='The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.', - examples=['Authorization', 'Api-Token', 'X-Auth-Token'], - title='Header Name', + description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", + examples=["Authorization", "Api-Token", "X-Auth-Token"], + title="Header Name", ) inject_into: Optional[RequestOption] = Field( None, - description='Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.', + description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", examples=[ - {'inject_into': 'header', 'field_name': 'Authorization'}, - {'inject_into': 'request_parameter', 'field_name': 'authKey'}, + {"inject_into": "header", "field_name": "Authorization"}, + {"inject_into": "request_parameter", "field_name": "authKey"}, ], - title='Inject API Key Into Outgoing HTTP Request', + title="Inject API Key Into Outgoing HTTP Request", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class AuthFlow(BaseModel): - auth_flow_type: Optional[AuthFlowType] = Field( - None, description='The type of auth to use', title='Auth flow type' - ) + auth_flow_type: Optional[AuthFlowType] = Field(None, description="The type of auth to use", title="Auth flow type") predicate_key: Optional[List[str]] = Field( None, - description='JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.', - examples=[['credentials', 'auth_type']], - title='Predicate key', + description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", + examples=[["credentials", "auth_type"]], + title="Predicate key", ) predicate_value: Optional[str] = Field( None, - description='Value of the predicate_key fields for the advanced auth to be applicable.', - examples=['Oauth'], - title='Predicate value', + description="Value of the predicate_key fields for the advanced auth to be applicable.", + examples=["Oauth"], + title="Predicate value", ) oauth_config_specification: Optional[OAuthConfigSpecification] = None class CursorPagination(BaseModel): - type: Literal['CursorPagination'] + type: Literal["CursorPagination"] cursor_value: str = Field( ..., - description='Value of the cursor defining the next page to fetch.', + description="Value of the cursor defining the next page to fetch.", examples=[ - '{{ headers.link.next.cursor }}', + "{{ headers.link.next.cursor }}", "{{ last_records[-1]['key'] }}", "{{ response['nextPage'] }}", ], - title='Cursor Value', + title="Cursor Value", ) page_size: Optional[int] = Field( None, - description='The number of records to include in each pages.', + description="The number of records to include in each pages.", examples=[100], - title='Page Size', + title="Page Size", ) stop_condition: Optional[str] = Field( None, - description='Template string evaluating when to stop paginating.', + description="Template string evaluating when to stop paginating.", examples=[ - '{{ response.data.has_more is false }}', + "{{ response.data.has_more is false }}", "{{ 'next' not in headers['link'] }}", ], - title='Stop Condition', + title="Stop Condition", ) decoder: Optional[JsonDecoder] = Field( None, - description='Component decoding the response so records can be extracted.', - title='Decoder', + description="Component decoding the response so records can be extracted.", + title="Decoder", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DatetimeBasedCursor(BaseModel): - type: Literal['DatetimeBasedCursor'] + type: Literal["DatetimeBasedCursor"] cursor_field: str = Field( ..., - description='The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.', - examples=['created_at', "{{ config['record_cursor'] }}"], - title='Cursor Field', + description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", + examples=["created_at", "{{ config['record_cursor'] }}"], + title="Cursor Field", ) datetime_format: str = Field( ..., - description='The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', - examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s', '%ms'], - title='Outgoing Datetime Format', + description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", + examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms"], + title="Outgoing Datetime Format", ) start_datetime: Union[str, MinMaxDatetime] = Field( ..., - description='The datetime that determines the earliest record that should be synced.', - examples=['2020-01-1T00:00:00Z', "{{ config['start_time'] }}"], - title='Start Datetime', + description="The datetime that determines the earliest record that should be synced.", + examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], + title="Start Datetime", ) cursor_datetime_formats: Optional[List[str]] = Field( None, - description='The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.', - title='Cursor Datetime Formats', + description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", + title="Cursor Datetime Formats", ) cursor_granularity: Optional[str] = Field( None, - description='Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.', - examples=['PT1S'], - title='Cursor Granularity', + description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", + examples=["PT1S"], + title="Cursor Granularity", ) end_datetime: Optional[Union[str, MinMaxDatetime]] = Field( None, - description='The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.', - examples=['2021-01-1T00:00:00Z', '{{ now_utc() }}', '{{ day_delta(-1) }}'], - title='End Datetime', + description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", + examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], + title="End Datetime", ) end_time_option: Optional[RequestOption] = Field( None, - description='Optionally configures how the end datetime will be sent in requests to the source API.', - title='Inject End Time Into Outgoing HTTP Request', + description="Optionally configures how the end datetime will be sent in requests to the source API.", + title="Inject End Time Into Outgoing HTTP Request", ) is_data_feed: Optional[bool] = Field( None, - description='A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.', - title='Whether the target API is formatted as a data feed', + description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", + title="Whether the target API is formatted as a data feed", ) lookback_window: Optional[str] = Field( None, - description='Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.', - examples=['P1D', "P{{ config['lookback_days'] }}D"], - title='Lookback Window', + description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", + examples=["P1D", "P{{ config['lookback_days'] }}D"], + title="Lookback Window", ) partition_field_end: Optional[str] = Field( None, - description='Name of the partition start time field.', - examples=['ending_time'], - title='Partition Field End', + description="Name of the partition start time field.", + examples=["ending_time"], + title="Partition Field End", ) partition_field_start: Optional[str] = Field( None, - description='Name of the partition end time field.', - examples=['starting_time'], - title='Partition Field Start', + description="Name of the partition end time field.", + examples=["starting_time"], + title="Partition Field Start", ) start_time_option: Optional[RequestOption] = Field( None, - description='Optionally configures how the start datetime will be sent in requests to the source API.', - title='Inject Start Time Into Outgoing HTTP Request', + description="Optionally configures how the start datetime will be sent in requests to the source API.", + title="Inject Start Time Into Outgoing HTTP Request", ) step: Optional[str] = Field( None, - description='The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.', - examples=['P1W', "{{ config['step_increment'] }}"], - title='Step', + description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", + examples=["P1W", "{{ config['step_increment'] }}"], + title="Step", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DefaultErrorHandler(BaseModel): - type: Literal['DefaultErrorHandler'] + type: Literal["DefaultErrorHandler"] backoff_strategies: Optional[ List[ Union[ @@ -897,144 +889,142 @@ class DefaultErrorHandler(BaseModel): ] ] = Field( None, - description='List of backoff strategies to use to determine how long to wait before retrying a retryable request.', - title='Backoff Strategies', + description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", + title="Backoff Strategies", ) max_retries: Optional[int] = Field( 5, - description='The maximum number of time to retry a retryable request before giving up and failing.', + description="The maximum number of time to retry a retryable request before giving up and failing.", examples=[5, 0, 10], - title='Max Retry Count', + title="Max Retry Count", ) response_filters: Optional[List[HttpResponseFilter]] = Field( None, description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", - title='Response Filters', + title="Response Filters", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DefaultPaginator(BaseModel): - type: Literal['DefaultPaginator'] - pagination_strategy: Union[ - CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement - ] = Field( + type: Literal["DefaultPaginator"] + pagination_strategy: Union[CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement] = Field( ..., - description='Strategy defining how records are paginated.', - title='Pagination Strategy', + description="Strategy defining how records are paginated.", + title="Pagination Strategy", ) decoder: Optional[JsonDecoder] = Field( None, - description='Component decoding the response so records can be extracted.', - title='Decoder', + description="Component decoding the response so records can be extracted.", + title="Decoder", ) page_size_option: Optional[RequestOption] = None page_token_option: Optional[Union[RequestOption, RequestPath]] = None - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DpathExtractor(BaseModel): - type: Literal['DpathExtractor'] + type: Literal["DpathExtractor"] field_path: List[str] = Field( ..., description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', examples=[ - ['data'], - ['data', 'records'], - ['data', '{{ parameters.name }}'], - ['data', '*', 'record'], + ["data"], + ["data", "records"], + ["data", "{{ parameters.name }}"], + ["data", "*", "record"], ], - title='Field Path', + title="Field Path", ) decoder: Optional[JsonDecoder] = Field( None, - description='Component decoding the response so records can be extracted.', - title='Decoder', + description="Component decoding the response so records can be extracted.", + title="Decoder", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SessionTokenRequestApiKeyAuthenticator(BaseModel): - type: Literal['ApiKey'] + type: Literal["ApiKey"] inject_into: RequestOption = Field( ..., - description='Configure how the API Key will be sent in requests to the source API.', + description="Configure how the API Key will be sent in requests to the source API.", examples=[ - {'inject_into': 'header', 'field_name': 'Authorization'}, - {'inject_into': 'request_parameter', 'field_name': 'authKey'}, + {"inject_into": "header", "field_name": "Authorization"}, + {"inject_into": "request_parameter", "field_name": "authKey"}, ], - title='Inject API Key Into Outgoing HTTP Request', + title="Inject API Key Into Outgoing HTTP Request", ) class ListPartitionRouter(BaseModel): - type: Literal['ListPartitionRouter'] + type: Literal["ListPartitionRouter"] cursor_field: str = Field( ..., description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', - examples=['section', "{{ config['section_key'] }}"], - title='Current Partition Value Identifier', + examples=["section", "{{ config['section_key'] }}"], + title="Current Partition Value Identifier", ) values: Union[str, List[str]] = Field( ..., - description='The list of attributes being iterated over and used as input for the requests made to the source API.', - examples=[['section_a', 'section_b', 'section_c'], "{{ config['sections'] }}"], - title='Partition Values', + description="The list of attributes being iterated over and used as input for the requests made to the source API.", + examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], + title="Partition Values", ) request_option: Optional[RequestOption] = Field( None, - description='A request option describing where the list value should be injected into and under what field name if applicable.', - title='Inject Partition Value Into Outgoing HTTP Request', + description="A request option describing where the list value should be injected into and under what field name if applicable.", + title="Inject Partition Value Into Outgoing HTTP Request", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class RecordSelector(BaseModel): - type: Literal['RecordSelector'] + type: Literal["RecordSelector"] extractor: Union[CustomRecordExtractor, DpathExtractor] record_filter: Optional[RecordFilter] = Field( None, - description='Responsible for filtering records to be emitted by the Source.', - title='Record Filter', + description="Responsible for filtering records to be emitted by the Source.", + title="Record Filter", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class Spec(BaseModel): - type: Literal['Spec'] + type: Literal["Spec"] connection_specification: Dict[str, Any] = Field( ..., - description='A connection specification describing how a the connector can be configured.', - title='Connection Specification', + description="A connection specification describing how a the connector can be configured.", + title="Connection Specification", ) documentation_url: Optional[str] = Field( None, description="URL of the connector's documentation page.", - examples=['https://docs.airbyte.com/integrations/sources/dremio'], - title='Documentation URL', + examples=["https://docs.airbyte.com/integrations/sources/dremio"], + title="Documentation URL", ) advanced_auth: Optional[AuthFlow] = Field( None, - description='Advanced specification for configuring the authentication flow.', - title='Advanced Auth', + description="Advanced specification for configuring the authentication flow.", + title="Advanced Auth", ) class CompositeErrorHandler(BaseModel): - type: Literal['CompositeErrorHandler'] + type: Literal["CompositeErrorHandler"] error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( ..., - description='List of error handlers to iterate on to determine how to handle a failed response.', - title='Error Handlers', + description="List of error handlers to iterate on to determine how to handle a failed response.", + title="Error Handlers", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DeclarativeSource(BaseModel): class Config: extra = Extra.forbid - type: Literal['DeclarativeSource'] + type: Literal["DeclarativeSource"] check: CheckStream streams: List[DeclarativeStream] version: str @@ -1043,7 +1033,7 @@ class Config: spec: Optional[Spec] = None metadata: Optional[Dict[str, Any]] = Field( None, - description='For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.', + description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", ) @@ -1051,101 +1041,91 @@ class DeclarativeStream(BaseModel): class Config: extra = Extra.allow - type: Literal['DeclarativeStream'] + type: Literal["DeclarativeStream"] retriever: Union[CustomRetriever, SimpleRetriever] = Field( ..., - description='Component used to coordinate how records are extracted across stream slices and request pages.', - title='Retriever', + description="Component used to coordinate how records are extracted across stream slices and request pages.", + title="Retriever", ) - incremental_sync: Optional[ - Union[CustomIncrementalSync, DatetimeBasedCursor] - ] = Field( + incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = Field( None, - description='Component used to fetch data incrementally based on a time field in the data.', - title='Incremental Sync', - ) - name: Optional[str] = Field( - '', description='The stream name.', example=['Users'], title='Name' - ) - primary_key: Optional[PrimaryKey] = Field( - '', description='The primary key of the stream.', title='Primary Key' + description="Component used to fetch data incrementally based on a time field in the data.", + title="Incremental Sync", ) + name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") + primary_key: Optional[PrimaryKey] = Field("", description="The primary key of the stream.", title="Primary Key") schema_loader: Optional[Union[InlineSchemaLoader, JsonFileSchemaLoader]] = Field( None, - description='Component used to retrieve the schema for the current stream.', - title='Schema Loader', + description="Component used to retrieve the schema for the current stream.", + title="Schema Loader", ) - transformations: Optional[ - List[Union[AddFields, CustomTransformation, RemoveFields]] - ] = Field( + transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields]]] = Field( None, - description='A list of transformations to be applied to each output record.', - title='Transformations', + description="A list of transformations to be applied to each output record.", + title="Transformations", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SessionTokenAuthenticator(BaseModel): - type: Literal['SessionTokenAuthenticator'] + type: Literal["SessionTokenAuthenticator"] login_requester: HttpRequester = Field( ..., - description='Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.', + description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", examples=[ { - 'type': 'HttpRequester', - 'url_base': 'https://my_api.com', - 'path': '/login', - 'authenticator': { - 'type': 'BasicHttpAuthenticator', - 'username': '{{ config.username }}', - 'password': '{{ config.password }}', + "type": "HttpRequester", + "url_base": "https://my_api.com", + "path": "/login", + "authenticator": { + "type": "BasicHttpAuthenticator", + "username": "{{ config.username }}", + "password": "{{ config.password }}", }, } ], - title='Login Requester', + title="Login Requester", ) session_token_path: List[str] = Field( ..., - description='The path in the response body returned from the login requester to the session token.', - examples=[['access_token'], ['result', 'token']], - title='Session Token Path', + description="The path in the response body returned from the login requester to the session token.", + examples=[["access_token"], ["result", "token"]], + title="Session Token Path", ) expiration_duration: Optional[str] = Field( None, - description='The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.', - examples=['PT1H', 'P1D'], - title='Expiration Duration', + description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", + examples=["PT1H", "P1D"], + title="Expiration Duration", ) - request_authentication: Union[ - SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator - ] = Field( + request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator] = Field( ..., - description='Authentication method to use for requests sent to the API, specifying how to inject the session token.', - title='Data Request Authentication', + description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", + title="Data Request Authentication", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class HttpRequester(BaseModel): - type: Literal['HttpRequester'] + type: Literal["HttpRequester"] url_base: str = Field( ..., - description='Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.', + description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", examples=[ - 'https://connect.squareup.com/v2', + "https://connect.squareup.com/v2", "{{ config['base_url'] or 'https://app.posthog.com'}}/api/", ], - title='API Base URL', + title="API Base URL", ) path: str = Field( ..., - description='Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.', + description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", examples=[ - '/products', + "/products", "/quotes/{{ stream_partition['id'] }}/quote_line_groups", "/trades/{{ config['symbol_id'] }}/history", ], - title='URL Path', + title="URL Path", ) authenticator: Optional[ Union[ @@ -1160,96 +1140,92 @@ class HttpRequester(BaseModel): ] ] = Field( None, - description='Authentication method to use for requests sent to the API.', - title='Authenticator', + description="Authentication method to use for requests sent to the API.", + title="Authenticator", ) - error_handler: Optional[ - Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler] - ] = Field( + error_handler: Optional[Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler]] = Field( None, - description='Error handler component that defines how to handle errors.', - title='Error Handler', + description="Error handler component that defines how to handle errors.", + title="Error Handler", ) http_method: Optional[Union[str, HttpMethodEnum]] = Field( - 'GET', - description='The HTTP method used to fetch data from the source (can be GET or POST).', - examples=['GET', 'POST'], - title='HTTP Method', + "GET", + description="The HTTP method used to fetch data from the source (can be GET or POST).", + examples=["GET", "POST"], + title="HTTP Method", ) request_body_data: Optional[Union[str, Dict[str, str]]] = Field( None, - description='Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.', + description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", examples=[ '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' ], - title='Request Body Payload (Non-JSON)', + title="Request Body Payload (Non-JSON)", ) request_body_json: Optional[Union[str, Dict[str, Any]]] = Field( None, - description='Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.', + description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", examples=[ - {'sort_order': 'ASC', 'sort_field': 'CREATED_AT'}, - {'key': "{{ config['value'] }}"}, - {'sort': {'field': 'updated_at', 'order': 'ascending'}}, + {"sort_order": "ASC", "sort_field": "CREATED_AT"}, + {"key": "{{ config['value'] }}"}, + {"sort": {"field": "updated_at", "order": "ascending"}}, ], - title='Request Body JSON Payload', + title="Request Body JSON Payload", ) request_headers: Optional[Union[str, Dict[str, str]]] = Field( None, - description='Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.', - examples=[{'Output-Format': 'JSON'}, {'Version': "{{ config['version'] }}"}], - title='Request Headers', + description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", + examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], + title="Request Headers", ) request_parameters: Optional[Union[str, Dict[str, str]]] = Field( None, - description='Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.', + description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", examples=[ - {'unit': 'day'}, + {"unit": "day"}, { - 'query': 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' }, - {'searchIn': "{{ ','.join(config.get('search_in', [])) }}"}, - {'sort_by[asc]': 'updated_at'}, + {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, + {"sort_by[asc]": "updated_at"}, ], - title='Query Parameters', + title="Query Parameters", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class ParentStreamConfig(BaseModel): - type: Literal['ParentStreamConfig'] + type: Literal["ParentStreamConfig"] parent_key: str = Field( ..., - description='The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.', - examples=['id', "{{ config['parent_record_id'] }}"], - title='Parent Key', - ) - stream: DeclarativeStream = Field( - ..., description='Reference to the parent stream.', title='Parent Stream' + description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", + examples=["id", "{{ config['parent_record_id'] }}"], + title="Parent Key", ) + stream: DeclarativeStream = Field(..., description="Reference to the parent stream.", title="Parent Stream") partition_field: str = Field( ..., - description='While iterating over parent records during a sync, the parent_key value can be referenced by using this field.', - examples=['parent_id', "{{ config['parent_partition_field'] }}"], - title='Current Parent Key Value Identifier', + description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", + examples=["parent_id", "{{ config['parent_partition_field'] }}"], + title="Current Parent Key Value Identifier", ) request_option: Optional[RequestOption] = Field( None, - description='A request option describing where the parent key value should be injected into and under what field name if applicable.', - title='Request Option', + description="A request option describing where the parent key value should be injected into and under what field name if applicable.", + title="Request Option", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SimpleRetriever(BaseModel): - type: Literal['SimpleRetriever'] + type: Literal["SimpleRetriever"] record_selector: RecordSelector = Field( ..., - description='Component that describes how to extract records from a HTTP response.', + description="Component that describes how to extract records from a HTTP response.", ) requester: Union[CustomRequester, HttpRequester] = Field( ..., - description='Requester component that describes how to prepare HTTP requests to send to the source API.', + description="Requester component that describes how to prepare HTTP requests to send to the source API.", ) paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( None, @@ -1260,28 +1236,24 @@ class SimpleRetriever(BaseModel): CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter, - List[ - Union[ - CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter - ] - ], + List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]], ] ] = Field( [], - description='PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.', - title='Partition Router', + description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", + title="Partition Router", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SubstreamPartitionRouter(BaseModel): - type: Literal['SubstreamPartitionRouter'] + type: Literal["SubstreamPartitionRouter"] parent_stream_configs: List[ParentStreamConfig] = Field( ..., - description='Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.', - title='Parent Stream Configs', + description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", + title="Parent Stream Configs", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") CompositeErrorHandler.update_forward_refs() diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java index 9625259aa0077..0c5408b475ea5 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java @@ -19,15 +19,11 @@ import io.airbyte.integrations.destination.bigquery.uploader.BigQueryUploaderFactory; import io.airbyte.integrations.destination.bigquery.uploader.UploaderType; import io.airbyte.integrations.destination.bigquery.uploader.config.UploaderConfig; -import io.airbyte.integrations.destination.s3.avro.JsonToAvroSchemaConverter; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.io.IOException; import java.util.Map; -import java.util.function.BiFunction; -import java.util.function.Function; import javax.annotation.Nullable; -import org.apache.avro.Schema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAsyncFlush.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAsyncFlush.java index 3d21bf42b420e..fb9114b990bcd 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAsyncFlush.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAsyncFlush.java @@ -13,7 +13,6 @@ import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.StreamDescriptor; -import java.util.List; import java.util.Map; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index 09087c4cc530c..80373997df5c7 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -395,7 +395,7 @@ private TyperDeduper buildTyperDeduper(final BigQuerySqlGenerator sqlGenerator, parsedCatalog, migrator, v2RawTableMigrator, - 8); + 8); } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java index 21417c2927d6f..eaf077d986d74 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java @@ -133,29 +133,29 @@ public void copyIntoTableFromStage(final String datasetId, LOGGER.info("Uploading records from staging files to target table {} (dataset {}): {}", tableId, datasetId, stagedFileName); - final String fullFilePath = String.format("gs://%s/%s%s", gcsConfig.getBucketName(), getStagingFullPath(datasetId, stream), stagedFileName); - LOGGER.info("Uploading staged file: {}", fullFilePath); - final LoadJobConfiguration configuration = LoadJobConfiguration.builder(tableId, fullFilePath) - .setFormatOptions(FormatOptions.csv()) - .setSchema(tableSchema) - .setWriteDisposition(WriteDisposition.WRITE_APPEND) - .setJobTimeoutMs(60000L) - .build(); - - final Job loadJob = this.bigQuery.create(JobInfo.of(configuration)); - LOGGER.info("[{}] Created a new job to upload record(s) to target table {} (dataset {}): {}", loadJob.getJobId(), - tableId, datasetId, loadJob); - - try { - BigQueryUtils.waitForJobFinish(loadJob); - LOGGER.info("[{}] Target table {} (dataset {}) is successfully appended with staging files", loadJob.getJobId(), - tableId, datasetId); - } catch (final BigQueryException | InterruptedException e) { - throw new RuntimeException( - String.format("[%s] Failed to upload staging files to destination table %s (%s)", loadJob.getJobId(), - tableId, datasetId), - e); - } + final String fullFilePath = String.format("gs://%s/%s%s", gcsConfig.getBucketName(), getStagingFullPath(datasetId, stream), stagedFileName); + LOGGER.info("Uploading staged file: {}", fullFilePath); + final LoadJobConfiguration configuration = LoadJobConfiguration.builder(tableId, fullFilePath) + .setFormatOptions(FormatOptions.csv()) + .setSchema(tableSchema) + .setWriteDisposition(WriteDisposition.WRITE_APPEND) + .setJobTimeoutMs(60000L) + .build(); + + final Job loadJob = this.bigQuery.create(JobInfo.of(configuration)); + LOGGER.info("[{}] Created a new job to upload record(s) to target table {} (dataset {}): {}", loadJob.getJobId(), + tableId, datasetId, loadJob); + + try { + BigQueryUtils.waitForJobFinish(loadJob); + LOGGER.info("[{}] Target table {} (dataset {}) is successfully appended with staging files", loadJob.getJobId(), + tableId, datasetId); + } catch (final BigQueryException | InterruptedException e) { + throw new RuntimeException( + String.format("[%s] Failed to upload staging files to destination table %s (%s)", loadJob.getJobId(), + tableId, datasetId), + e); + } } @Override diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java index 73ce256e1dc14..9996b0d16711d 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java @@ -69,8 +69,8 @@ public SerializedAirbyteMessageConsumer createAsync( } /** - * Out BigQuery's uploader threads use a fair amount of memory. We believe this is largely - * due to the sdk client we use. + * Out BigQuery's uploader threads use a fair amount of memory. We believe this is largely due to + * the sdk client we use. * * @return number of bytes to make available for message buffering. */ @@ -79,10 +79,10 @@ private long getBigQueryBufferMemoryLimit() { } private Map createWriteConfigs(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final ParsedCatalog parsedCatalog, - final Function recordFormatterCreator, - final Function tmpTableNameTransformer) { + final ConfiguredAirbyteCatalog catalog, + final ParsedCatalog parsedCatalog, + final Function recordFormatterCreator, + final Function tmpTableNameTransformer) { return catalog.getStreams().stream() .map(configuredStream -> { Preconditions.checkNotNull(configuredStream.getDestinationSyncMode(), "Undefined destination sync mode"); diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryWriteConfig.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryWriteConfig.java index 148c7c95382ad..7e51dcdfce74d 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryWriteConfig.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryWriteConfig.java @@ -46,4 +46,5 @@ public BigQueryWriteConfig(final String streamName, tableSchema, syncMode); } + } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java index 7f27d5e83d6d5..affaef8a3d4dc 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java @@ -72,8 +72,7 @@ public void execute(final String sql) throws InterruptedException { if (job.getStatus().getError() != null) { throw new BigQueryException(Streams.concat( Stream.of(job.getStatus().getError()), - job.getStatus().getExecutionErrors().stream() - ).toList()); + job.getStatus().getExecutionErrors().stream()).toList()); } final JobStatistics.QueryStatistics statistics = job.getStatistics(); diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java index 3e5c73260a1d1..2d5a8644fa332 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java @@ -17,7 +17,6 @@ import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.nio.file.Path; import java.util.HashSet; - import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; diff --git a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py index 5b6c892dd7a24..6e3d0882130f0 100644 --- a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py +++ b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py @@ -14,14 +14,7 @@ import duckdb from airbyte_cdk import AirbyteLogger from airbyte_cdk.destinations import Destination -from airbyte_cdk.models import ( - AirbyteConnectionStatus, - AirbyteMessage, - ConfiguredAirbyteCatalog, - DestinationSyncMode, - Status, - Type, -) +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type logger = getLogger("airbyte") @@ -46,9 +39,7 @@ def _get_destination_path(destination_path: str) -> str: Get a normalized version of the destination path. Automatically append /local/ to the start of the path """ - if destination_path.startswith("md:") or destination_path.startswith( - "motherduck:" - ): + if destination_path.startswith("md:") or destination_path.startswith("motherduck:"): return destination_path if not destination_path.startswith("/local"): @@ -57,8 +48,7 @@ def _get_destination_path(destination_path: str) -> str: destination_path = os.path.normpath(destination_path) if not destination_path.startswith("/local"): raise ValueError( - f"destination_path={destination_path} is not a valid path." - "A valid path shall start with /local or no / prefix" + f"destination_path={destination_path} is not a valid path." "A valid path shall start with /local or no / prefix" ) return destination_path @@ -140,9 +130,7 @@ def write( data = message.record.data stream = message.record.stream if stream not in streams: - logger.debug( - f"Stream {stream} was not present in configured streams, skipping" - ) + logger.debug(f"Stream {stream} was not present in configured streams, skipping") continue # add to buffer @@ -167,9 +155,7 @@ def write( con.executemany(query, buffer[stream_name]) con.commit() - def check( - self, logger: AirbyteLogger, config: Mapping[str, Any] - ) -> AirbyteConnectionStatus: + def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: """ Tests if the input configuration can be used to successfully connect to the destination with the needed permissions e.g: if a provided API token or password can be used to connect and write to the destination. @@ -198,6 +184,4 @@ def check( return AirbyteConnectionStatus(status=Status.SUCCEEDED) except Exception as e: - return AirbyteConnectionStatus( - status=Status.FAILED, message=f"An exception occurred: {repr(e)}" - ) + return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}") diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java index 644f2c54e8570..e5c7339c28a49 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java @@ -85,7 +85,8 @@ public void execute(final String sql) throws Exception { database.execute(sql); } catch (final SnowflakeSQLException e) { LOGGER.error("Sql {} failed", queryId, e); - // Snowflake SQL exceptions by default may not be super helpful, so we try to extract the relevant part of the message. + // Snowflake SQL exceptions by default may not be super helpful, so we try to extract the relevant + // part of the message. final String trimmedMessage; if (e.getMessage().startsWith(EXCEPTION_COMMON_PREFIX)) { // The first line is a pretty generic message, so just remove it diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java index 7138ce1abb4d8..e0af9b0dcfbac 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java @@ -36,7 +36,8 @@ public class SnowflakeSqlGenerator implements SqlGenerator RESERVED_COLUMN_NAMES = ImmutableList.of( "CURRENT_DATE", "CURRENT_TIME", @@ -62,8 +63,8 @@ public ColumnId buildColumnId(final String name, final String suffix) { // No escaping needed, as far as I can tell. We quote all our identifier names. final String nameWithSuffix = name + suffix; return new ColumnId(prefixReservedColumnName(escapeSqlIdentifier(name).toUpperCase()) + suffix, - nameWithSuffix, - nameWithSuffix.toUpperCase()); + nameWithSuffix, + nameWithSuffix.toUpperCase()); } public String toDialectType(final AirbyteType type) { @@ -196,18 +197,22 @@ private String extractAndCast(final ColumnId column, final AirbyteType airbyteTy } /** - * The `${` bigram causes problems inside script blocks. For example, a perfectly innocuous query like - * `SELECT "_airbyte_data":"${foo}" FROM ...` works fine normally, but running this block will throw - * an error: - * 
{@code
+   * The `${` bigram causes problems inside script blocks. For example, a perfectly innocuous query
+   * like `SELECT "_airbyte_data":"${foo}" FROM ...` works fine normally, but running this block will
+   * throw an error:
+   *
+   * 
+   * {@code
    * EXECUTE IMMEDIATE 'BEGIN
    * LET x INTEGER := (SELECT "_airbyte_data":"${foo}" FROM ...);
    * END;';
-   * }

+   * }
+   *
*

- * This method is a workaround for this behavior. We switch to using the {@code get} method to extract - * JSON values, and avoid the `${` sequence by using string concatenation. This generates a sql statement - * like {@code SELECT TRY_CAST((get("_airbyte_data", '$' + '{foo}')::text) as INTEGER) FROM ...}. + * This method is a workaround for this behavior. We switch to using the {@code get} method to + * extract JSON values, and avoid the `${` sequence by using string concatenation. This generates a + * sql statement like {@code SELECT TRY_CAST((get("_airbyte_data", '$' + '{foo}')::text) as INTEGER) + * FROM ...}. */ private String extractAndCastInsideScript(final ColumnId column, final AirbyteType airbyteType) { final String[] parts = column.originalName().split("\\$\\{", -1); @@ -561,8 +566,8 @@ public String migrateFromV1toV2(final StreamId streamId, final String namespace, } /** - * Snowflake json object access is done using double-quoted strings, e.g. `SELECT "_airbyte_data":"foo"`. - * As such, we need to escape double-quotes in the field name. + * Snowflake json object access is done using double-quoted strings, e.g. `SELECT + * "_airbyte_data":"foo"`. As such, we need to escape double-quotes in the field name. */ public static String escapeJsonIdentifier(final String identifier) { // Note that we don't need to escape backslashes here! @@ -592,8 +597,7 @@ public static String escapeSqlIdentifier(String identifier) { } private static String prefixReservedColumnName(final String columnName) { - return RESERVED_COLUMN_NAMES.stream().anyMatch(k -> k.equalsIgnoreCase(columnName)) ? - "_" + columnName : columnName; + return RESERVED_COLUMN_NAMES.stream().anyMatch(k -> k.equalsIgnoreCase(columnName)) ? "_" + columnName : columnName; } public static String escapeSingleQuotedString(final String str) { diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java index 26c28944495a6..0cf13c579d138 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java @@ -37,7 +37,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import javax.sql.DataSource; -import net.snowflake.client.jdbc.SnowflakeSQLException; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringSubstitutor; import org.junit.jupiter.api.AfterAll; diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorTest.java index ed92be45193e5..49cc80abececd 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.snowflake.typing_deduping; import static org.junit.jupiter.api.Assertions.assertAll; @@ -19,20 +23,15 @@ void columnNameSpecialCharacterHandling() { new ColumnId( "__FOO_", "${foo}", - "${FOO}" - ), - generator.buildColumnId("${foo}") - ), + "${FOO}"), + generator.buildColumnId("${foo}")), // But normally, we should leave those characters untouched. () -> assertEquals( new ColumnId( "{FO$O}", "{fo$o}", - "{FO$O}" - ), - generator.buildColumnId("{fo$o}") - ) - ); + "{FO$O}"), + generator.buildColumnId("{fo$o}"))); } /** @@ -48,10 +47,8 @@ void streamNameSpecialCharacterHandling() { "airbyte_internal", "__foo__raw__stream___bar_", "${foo}", - "${bar}" - ), - generator.buildStreamId("${foo}", "${bar}", "airbyte_internal") - ), + "${bar}"), + generator.buildStreamId("${foo}", "${bar}", "airbyte_internal")), () -> assertEquals( new StreamId( "{FO$O}", @@ -59,10 +56,8 @@ void streamNameSpecialCharacterHandling() { "airbyte_internal", "{fo$o}_raw__stream_{ba$r}", "{fo$o}", - "{ba$r}" - ), - generator.buildStreamId("{fo$o}", "{ba$r}", "airbyte_internal") - ) - ); + "{ba$r}"), + generator.buildStreamId("{fo$o}", "{ba$r}", "airbyte_internal"))); } + } diff --git a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml index d7b2322d7c7f5..4b33494571475 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml +++ b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml @@ -7,7 +7,7 @@ data: enabled: true cloud: enabled: true - dockerImageTag: 0.2.0 # https://github.com/airbytehq/airbyte/issues/30478 + dockerImageTag: 0.2.0 # https://github.com/airbytehq/airbyte/issues/30478 connectorSubtype: api connectorType: source definitionId: 47f17145-fe20-4ef5-a548-e29b048adf84 diff --git a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/configured_catalog.json index 9eb524b550d75..2b0206c69c64f 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/configured_catalog.json @@ -4,191 +4,121 @@ "stream": { "name": "customers", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "date_modified" - ] + "default_cursor_field": ["date_modified"] }, "sync_mode": "incremental", - "cursor_field": [ - "date_modified" - ], + "cursor_field": ["date_modified"], "destination_sync_mode": "append" }, { "stream": { "name": "orders", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "date_modified" - ] + "default_cursor_field": ["date_modified"] }, "sync_mode": "incremental", - "cursor_field": [ - "date_modified" - ], + "cursor_field": ["date_modified"], "destination_sync_mode": "append" }, { "stream": { "name": "transactions", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "id" - ] + "default_cursor_field": ["id"] }, "sync_mode": "incremental", - "cursor_field": [ - "id" - ], + "cursor_field": ["id"], "destination_sync_mode": "append" }, { "stream": { "name": "pages", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "id" - ] + "default_cursor_field": ["id"] }, "sync_mode": "incremental", - "cursor_field": [ - "id" - ], + "cursor_field": ["id"], "destination_sync_mode": "append" }, { "stream": { "name": "brands", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "id" - ] + "default_cursor_field": ["id"] }, "sync_mode": "incremental", - "cursor_field": [ - "id" - ], + "cursor_field": ["id"], "destination_sync_mode": "append" }, { "stream": { "name": "categories", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "id" - ] + "default_cursor_field": ["id"] }, "sync_mode": "incremental", - "cursor_field": [ - "id" - ], + "cursor_field": ["id"], "destination_sync_mode": "append" }, { "stream": { "name": "products", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "date_modified" - ] + "default_cursor_field": ["date_modified"] }, "sync_mode": "incremental", - "cursor_field": [ - "date_modified" - ], + "cursor_field": ["date_modified"], "destination_sync_mode": "append" }, { "stream": { "name": "channels", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "date_modified" - ] + "default_cursor_field": ["date_modified"] }, "sync_mode": "incremental", - "cursor_field": [ - "date_modified" - ], + "cursor_field": ["date_modified"], "destination_sync_mode": "append" }, { "stream": { "name": "store", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "store_id" - ] + "default_cursor_field": ["store_id"] }, "sync_mode": "incremental", - "cursor_field": [ - "store_id" - ], + "cursor_field": ["store_id"], "destination_sync_mode": "overwrite" }, { "stream": { "name": "order_products", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "id" - ] + "default_cursor_field": ["id"] }, "sync_mode": "incremental", - "cursor_field": [ - "id" - ], + "cursor_field": ["id"], "destination_sync_mode": "append" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/channels.json b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/channels.json index 6e6e1d09f59c0..78e6edc8221ed 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/channels.json +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/channels.json @@ -36,7 +36,7 @@ }, "icon_url": { "type": ["null", "string"] - }, + }, "is_enabled": { "type": ["null", "boolean"] } diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/pages.json b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/pages.json index 00fded5a83339..003984b52eeaf 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/pages.json +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/pages.json @@ -58,6 +58,6 @@ }, "id": { "type": ["null", "integer"] - } + } } } diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json index b2f6edc474783..ce96ff0a6a99d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json @@ -28,7 +28,7 @@ "bid_strategy": { "type": ["null", "string"] }, - "boosted_object_id":{ + "boosted_object_id": { "type": ["null", "string"] }, "budget_rebalance_flag": { diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json index c812570a11d4f..4601d3d2a5103 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json @@ -6,12 +6,22 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "source_defined_primary_key": [["ad_group.id"], ["ad_group_ad.ad.id"], ["segments.date"], ["segments.ad_network_type"]], + "source_defined_primary_key": [ + ["ad_group.id"], + ["ad_group_ad.ad.id"], + ["segments.date"], + ["segments.ad_network_type"] + ], "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", - "primary_key": [["ad_group.id"], ["ad_group_ad.ad.id"], ["segments.date"], ["segments.ad_network_type"]], + "primary_key": [ + ["ad_group.id"], + ["ad_group_ad.ad.id"], + ["segments.date"], + ["segments.ad_network_type"] + ], "cursor_field": ["segments.date"] }, { @@ -22,23 +32,23 @@ "source_defined_cursor": true, "default_cursor_field": ["segments.date"], "source_defined_primary_key": [ - ["customer.id"], - ["campaign_budget.id"], - ["segments.date"], - ["segments.budget_campaign_association_status.campaign"], - ["segments.budget_campaign_association_status.status"] + ["customer.id"], + ["campaign_budget.id"], + ["segments.date"], + ["segments.budget_campaign_association_status.campaign"], + ["segments.budget_campaign_association_status.status"] ] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"], "primary_key": [ - ["customer.id"], - ["campaign_budget.id"], - ["segments.date"], - ["segments.budget_campaign_association_status.campaign"], - ["segments.budget_campaign_association_status.status"] - ] + ["customer.id"], + ["campaign_budget.id"], + ["segments.date"], + ["segments.budget_campaign_association_status.campaign"], + ["segments.budget_campaign_association_status.status"] + ] }, { "stream": { @@ -57,13 +67,23 @@ "name": "account_performance_report", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["customer.id"], ["segments.date"], ["segments.ad_network_type"], ["segments.device"]], + "source_defined_primary_key": [ + ["customer.id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", - "primary_key": [["customer.id"], ["segments.date"], ["segments.ad_network_type"], ["segments.device"]], + "primary_key": [ + ["customer.id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "cursor_field": ["segments.date"] }, { @@ -72,26 +92,44 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "source_defined_primary_key": [["click_view.gclid"], ["segments.date"], ["segments.ad_network_type"]], + "source_defined_primary_key": [ + ["click_view.gclid"], + ["segments.date"], + ["segments.ad_network_type"] + ], "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"], - "primary_key": [["click_view.gclid"], ["segments.date"], ["segments.ad_network_type"]] + "primary_key": [ + ["click_view.gclid"], + ["segments.date"], + ["segments.ad_network_type"] + ] }, { "stream": { "name": "geographic_report", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["customer.id"], ["geographic_view.country_criterion_id"], ["geographic_view.location_type"], ["segments.date"]], + "source_defined_primary_key": [ + ["customer.id"], + ["geographic_view.country_criterion_id"], + ["geographic_view.location_type"], + ["segments.date"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", - "primary_key": [["customer.id"], ["geographic_view.country_criterion_id"], ["geographic_view.location_type"], ["segments.date"]], + "primary_key": [ + ["customer.id"], + ["geographic_view.country_criterion_id"], + ["geographic_view.location_type"], + ["segments.date"] + ], "cursor_field": ["segments.date"] }, { @@ -99,13 +137,21 @@ "name": "keyword_report", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["ad_group_criterion.criterion_id"], ["ad_group.id"], ["segments.date"]], + "source_defined_primary_key": [ + ["ad_group_criterion.criterion_id"], + ["ad_group.id"], + ["segments.date"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", - "primary_key": [["ad_group_criterion.criterion_id"], ["ad_group.id"], ["segments.date"]], + "primary_key": [ + ["ad_group_criterion.criterion_id"], + ["ad_group.id"], + ["segments.date"] + ], "cursor_field": ["segments.date"] }, { @@ -113,13 +159,25 @@ "name": "display_keyword_performance_report", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["ad_group.id"], ["ad_group_criterion.criterion_id"], ["segments.date"], ["segments.ad_network_type"], ["segments.device"]], + "source_defined_primary_key": [ + ["ad_group.id"], + ["ad_group_criterion.criterion_id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", - "primary_key": [["ad_group.id"], ["ad_group_criterion.criterion_id"], ["segments.date"], ["segments.ad_network_type"], ["segments.device"]], + "primary_key": [ + ["ad_group.id"], + ["ad_group_criterion.criterion_id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "cursor_field": ["segments.date"] }, { @@ -127,12 +185,24 @@ "name": "display_topics_performance_report", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["ad_group.id"], ["ad_group_criterion.criterion_id"], ["segments.date"], ["segments.ad_network_type"], ["segments.device"]], + "source_defined_primary_key": [ + ["ad_group.id"], + ["ad_group_criterion.criterion_id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", - "primary_key": [["ad_group.id"], ["ad_group_criterion.criterion_id"], ["segments.date"], ["segments.ad_network_type"], ["segments.device"]], + "primary_key": [ + ["ad_group.id"], + ["ad_group_criterion.criterion_id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"] }, @@ -154,7 +224,11 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "source_defined_primary_key": [["ad_group.id"], ["ad_group_ad.ad.id"], ["segments.date"]], + "source_defined_primary_key": [ + ["ad_group.id"], + ["ad_group_ad.ad.id"], + ["segments.date"] + ], "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", @@ -197,12 +271,22 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["segments.date"], - "source_defined_primary_key": [["campaign.id"], ["segments.date"], ["segments.hour"], ["segments.ad_network_type"]] + "source_defined_primary_key": [ + ["campaign.id"], + ["segments.date"], + ["segments.hour"], + ["segments.ad_network_type"] + ] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"], - "primary_key": [["campaign.id"], ["segments.date"], ["segments.hour"], ["segments.ad_network_type"]] + "primary_key": [ + ["campaign.id"], + ["segments.date"], + ["segments.hour"], + ["segments.ad_network_type"] + ] }, { "stream": { @@ -231,7 +315,11 @@ "name": "ad_group_ad_labels", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["ad_group.id"], ["ad_group_ad.ad.id"], ["label.id"]] + "source_defined_primary_key": [ + ["ad_group.id"], + ["ad_group_ad.ad.id"], + ["label.id"] + ] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", @@ -243,12 +331,24 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "source_defined_primary_key": [["customer.id"], ["user_location_view.country_criterion_id"], ["user_location_view.targeting_location"], ["segments.date"], ["segments.ad_network_type"]], + "source_defined_primary_key": [ + ["customer.id"], + ["user_location_view.country_criterion_id"], + ["user_location_view.targeting_location"], + ["segments.date"], + ["segments.ad_network_type"] + ], "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", - "primary_key": [["customer.id"], ["user_location_view.country_criterion_id"], ["user_location_view.targeting_location"], ["segments.date"], ["segments.ad_network_type"]], + "primary_key": [ + ["customer.id"], + ["user_location_view.country_criterion_id"], + ["user_location_view.targeting_location"], + ["segments.date"], + ["segments.ad_network_type"] + ], "cursor_field": ["segments.date"] }, { @@ -321,13 +421,21 @@ "name": "campaign_bidding_strategies", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["campaign.id"], ["bidding_strategy.id"], ["segments.date"]], + "source_defined_primary_key": [ + ["campaign.id"], + ["bidding_strategy.id"], + ["segments.date"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", - "primary_key": [["campaign.id"], ["bidding_strategy.id"], ["segments.date"]], + "primary_key": [ + ["campaign.id"], + ["bidding_strategy.id"], + ["segments.date"] + ], "cursor_field": ["segments.date"] }, { @@ -335,13 +443,21 @@ "name": "ad_group_bidding_strategies", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["ad_group.id"], ["bidding_strategy.id"], ["segments.date"]], + "source_defined_primary_key": [ + ["ad_group.id"], + ["bidding_strategy.id"], + ["segments.date"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", - "primary_key": [["ad_group.id"], ["bidding_strategy.id"], ["segments.date"]], + "primary_key": [ + ["ad_group.id"], + ["bidding_strategy.id"], + ["segments.date"] + ], "cursor_field": ["segments.date"] }, { @@ -371,7 +487,9 @@ "name": "ad_group_criterion_labels", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["ad_group_criterion_label.resource_name"]] + "source_defined_primary_key": [ + ["ad_group_criterion_label.resource_name"] + ] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/state.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/state.json index 95d5f37563474..ce29ca2dbd2d7 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/state.json @@ -40,7 +40,7 @@ } } }, - { + { "type": "STREAM", "stream": { "stream_descriptor": { diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/abnormal_state.json index bf47ea0d96902..5e20a30c4498f 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/abnormal_state.json @@ -263,7 +263,7 @@ } } }, - { + { "type": "STREAM", "stream": { "stream_descriptor": { diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json index 842b35eff4f1b..8f212a2ae02bb 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json @@ -819,7 +819,11 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["yearWeek"], - "source_defined_primary_key": [["property_id"], ["yearWeek"], ["eventName"]] + "source_defined_primary_key": [ + ["property_id"], + ["yearWeek"], + ["eventName"] + ] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/defaults/default_reports.json b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/defaults/default_reports.json index ee9f5e39d8f07..559c52f37852b 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/defaults/default_reports.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/defaults/default_reports.json @@ -264,50 +264,24 @@ ] }, { - "name" : "events_report", - "dimensions" : [ - "date", - "eventName" - ], - "metrics" : [ - "eventCount", - "totalUsers", - "eventCountPerUser", - "totalRevenue" - ] + "name": "events_report", + "dimensions": ["date", "eventName"], + "metrics": ["eventCount", "totalUsers", "eventCountPerUser", "totalRevenue"] }, { - "name" : "weekly_events_report", - "dimensions" : [ - "yearWeek", - "eventName" - ], - "metrics" : [ - "eventCount", - "totalUsers", - "eventCountPerUser", - "totalRevenue" - ] + "name": "weekly_events_report", + "dimensions": ["yearWeek", "eventName"], + "metrics": ["eventCount", "totalUsers", "eventCountPerUser", "totalRevenue"] }, { - "name" : "conversions_report", - "dimensions" : [ - "date", - "eventName" - ], - "metrics" : [ - "conversions", - "totalUsers", - "totalRevenue" - ] + "name": "conversions_report", + "dimensions": ["date", "eventName"], + "metrics": ["conversions", "totalUsers", "totalRevenue"] }, { - "name" : "pages_title_and_screen_class_report", - "dimensions" : [ - "date", - "unifiedScreenClass" - ], - "metrics" : [ + "name": "pages_title_and_screen_class_report", + "dimensions": ["date", "unifiedScreenClass"], + "metrics": [ "screenPageViews", "totalUsers", "newUsers", @@ -318,12 +292,9 @@ ] }, { - "name" : "pages_path_report", - "dimensions" : [ - "date", - "pagePath" - ], - "metrics" : [ + "name": "pages_path_report", + "dimensions": ["date", "pagePath"], + "metrics": [ "screenPageViews", "totalUsers", "newUsers", @@ -334,12 +305,9 @@ ] }, { - "name" : "pages_title_and_screen_name_report", - "dimensions" : [ - "date", - "unifiedScreenName" - ], - "metrics" : [ + "name": "pages_title_and_screen_name_report", + "dimensions": ["date", "unifiedScreenName"], + "metrics": [ "screenPageViews", "totalUsers", "newUsers", @@ -350,12 +318,9 @@ ] }, { - "name" : "content_group_report", - "dimensions" : [ - "date", - "contentGroup" - ], - "metrics" : [ + "name": "content_group_report", + "dimensions": ["date", "contentGroup"], + "metrics": [ "screenPageViews", "totalUsers", "newUsers", @@ -366,12 +331,9 @@ ] }, { - "name" : "ecommerce_purchases_item_name_report", - "dimensions" : [ - "date", - "itemName" - ], - "metrics" : [ + "name": "ecommerce_purchases_item_name_report", + "dimensions": ["date", "itemName"], + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -381,12 +343,9 @@ ] }, { - "name" : "ecommerce_purchases_item_id_report", - "dimensions" : [ - "date", - "itemId" - ], - "metrics" : [ + "name": "ecommerce_purchases_item_id_report", + "dimensions": ["date", "itemId"], + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -396,8 +355,8 @@ ] }, { - "name" : "ecommerce_purchases_item_category_report_combined", - "dimensions" : [ + "name": "ecommerce_purchases_item_category_report_combined", + "dimensions": [ "date", "itemCategory", "itemCategory2", @@ -405,7 +364,7 @@ "itemCategory4", "itemCategory5" ], - "metrics" : [ + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -415,12 +374,9 @@ ] }, { - "name" : "ecommerce_purchases_item_category_report", - "dimensions" : [ - "date", - "itemCategory" - ], - "metrics" : [ + "name": "ecommerce_purchases_item_category_report", + "dimensions": ["date", "itemCategory"], + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -430,12 +386,9 @@ ] }, { - "name" : "ecommerce_purchases_item_category_2_report", - "dimensions" : [ - "date", - "itemCategory2" - ], - "metrics" : [ + "name": "ecommerce_purchases_item_category_2_report", + "dimensions": ["date", "itemCategory2"], + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -445,12 +398,9 @@ ] }, { - "name" : "ecommerce_purchases_item_category_3_report", - "dimensions" : [ - "date", - "itemCategory3" - ], - "metrics" : [ + "name": "ecommerce_purchases_item_category_3_report", + "dimensions": ["date", "itemCategory3"], + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -460,12 +410,9 @@ ] }, { - "name" : "ecommerce_purchases_item_category_4_report", - "dimensions" : [ - "date", - "itemCategory4" - ], - "metrics" : [ + "name": "ecommerce_purchases_item_category_4_report", + "dimensions": ["date", "itemCategory4"], + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -475,12 +422,9 @@ ] }, { - "name" : "ecommerce_purchases_item_category_5_report", - "dimensions" : [ - "date", - "itemCategory5" - ], - "metrics" : [ + "name": "ecommerce_purchases_item_category_5_report", + "dimensions": ["date", "itemCategory5"], + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -490,12 +434,9 @@ ] }, { - "name" : "ecommerce_purchases_item_brand_report", - "dimensions" : [ - "date", - "itemBrand" - ], - "metrics" : [ + "name": "ecommerce_purchases_item_brand_report", + "dimensions": ["date", "itemBrand"], + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -505,12 +446,9 @@ ] }, { - "name" : "publisher_ads_ad_unit_report", - "dimensions" : [ - "date", - "adUnitName" - ], - "metrics" : [ + "name": "publisher_ads_ad_unit_report", + "dimensions": ["date", "adUnitName"], + "metrics": [ "publisherAdImpressions", "adUnitExposure", "publisherAdClicks", @@ -518,12 +456,9 @@ ] }, { - "name" : "publisher_ads_page_path_report", - "dimensions" : [ - "date", - "pagePath" - ], - "metrics" : [ + "name": "publisher_ads_page_path_report", + "dimensions": ["date", "pagePath"], + "metrics": [ "publisherAdImpressions", "adUnitExposure", "publisherAdClicks", @@ -531,12 +466,9 @@ ] }, { - "name" : "publisher_ads_ad_format_report", - "dimensions" : [ - "date", - "adFormat" - ], - "metrics" : [ + "name": "publisher_ads_ad_format_report", + "dimensions": ["date", "adFormat"], + "metrics": [ "publisherAdImpressions", "adUnitExposure", "publisherAdClicks", @@ -544,12 +476,9 @@ ] }, { - "name" : "publisher_ads_ad_source_report", - "dimensions" : [ - "date", - "adSourceName" - ], - "metrics" : [ + "name": "publisher_ads_ad_source_report", + "dimensions": ["date", "adSourceName"], + "metrics": [ "publisherAdImpressions", "adUnitExposure", "publisherAdClicks", @@ -557,12 +486,9 @@ ] }, { - "name" : "demographic_country_report", - "dimensions" : [ - "date", - "country" - ], - "metrics" : [ + "name": "demographic_country_report", + "dimensions": ["date", "country"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -572,12 +498,9 @@ ] }, { - "name" : "demographic_region_report", - "dimensions" : [ - "date", - "region" - ], - "metrics" : [ + "name": "demographic_region_report", + "dimensions": ["date", "region"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -587,12 +510,9 @@ ] }, { - "name" : "demographic_city_report", - "dimensions" : [ - "date", - "city" - ], - "metrics" : [ + "name": "demographic_city_report", + "dimensions": ["date", "city"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -602,12 +522,9 @@ ] }, { - "name" : "demographic_language_report", - "dimensions" : [ - "date", - "language" - ], - "metrics" : [ + "name": "demographic_language_report", + "dimensions": ["date", "language"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -617,12 +534,9 @@ ] }, { - "name" : "demographic_age_report", - "dimensions" : [ - "date", - "userAgeBracket" - ], - "metrics" : [ + "name": "demographic_age_report", + "dimensions": ["date", "userAgeBracket"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -632,12 +546,9 @@ ] }, { - "name" : "demographic_gender_report", - "dimensions" : [ - "date", - "userGender" - ], - "metrics" : [ + "name": "demographic_gender_report", + "dimensions": ["date", "userGender"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -647,12 +558,9 @@ ] }, { - "name" : "demographic_interest_report", - "dimensions" : [ - "date", - "brandingInterest" - ], - "metrics" : [ + "name": "demographic_interest_report", + "dimensions": ["date", "brandingInterest"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -662,12 +570,9 @@ ] }, { - "name" : "tech_browser_report", - "dimensions" : [ - "date", - "browser" - ], - "metrics" : [ + "name": "tech_browser_report", + "dimensions": ["date", "browser"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -678,12 +583,9 @@ ] }, { - "name" : "tech_device_category_report", - "dimensions" : [ - "date", - "deviceCategory" - ], - "metrics" : [ + "name": "tech_device_category_report", + "dimensions": ["date", "deviceCategory"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -694,12 +596,9 @@ ] }, { - "name" : "tech_device_model_report", - "dimensions" : [ - "date", - "deviceModel" - ], - "metrics" : [ + "name": "tech_device_model_report", + "dimensions": ["date", "deviceModel"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -710,12 +609,9 @@ ] }, { - "name" : "tech_screen_resolution_report", - "dimensions" : [ - "date", - "screenResolution" - ], - "metrics" : [ + "name": "tech_screen_resolution_report", + "dimensions": ["date", "screenResolution"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -726,12 +622,9 @@ ] }, { - "name" : "tech_app_version_report", - "dimensions" : [ - "date", - "appVersion" - ], - "metrics" : [ + "name": "tech_app_version_report", + "dimensions": ["date", "appVersion"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -742,12 +635,9 @@ ] }, { - "name" : "tech_platform_report", - "dimensions" : [ - "date", - "platform" - ], - "metrics" : [ + "name": "tech_platform_report", + "dimensions": ["date", "platform"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -758,13 +648,9 @@ ] }, { - "name" : "tech_platform_device_category_report", - "dimensions" : [ - "date", - "platform", - "deviceCategory" - ], - "metrics" : [ + "name": "tech_platform_device_category_report", + "dimensions": ["date", "platform", "deviceCategory"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -775,12 +661,9 @@ ] }, { - "name" : "tech_operating_system_report", - "dimensions" : [ - "date", - "operatingSystem" - ], - "metrics" : [ + "name": "tech_operating_system_report", + "dimensions": ["date", "operatingSystem"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -791,12 +674,9 @@ ] }, { - "name" : "tech_os_with_version_report", - "dimensions" : [ - "date", - "operatingSystemWithVersion" - ], - "metrics" : [ + "name": "tech_os_with_version_report", + "dimensions": ["date", "operatingSystemWithVersion"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json index 487f0f84b0a98..0d1074e212d03 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json @@ -215,7 +215,7 @@ "sync_mode": "full_refresh", "destination_sync_mode": "append" }, - { + { "stream": { "name": "issue_transitions", "json_schema": {}, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/filter_sharing.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/filter_sharing.json index 44c717228f56d..e3c19d2e8876b 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/filter_sharing.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/filter_sharing.json @@ -8,8 +8,8 @@ "readOnly": true }, "filterId": { - "type" : ["null", "string"], - "description" : "Id of the related filter" + "type": ["null", "string"], + "description": "Id of the related filter" }, "type": { "type": "string", diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_comments.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_comments.json index a9202899f541d..538bf16bfac62 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_comments.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_comments.json @@ -13,7 +13,7 @@ "readOnly": true }, "issueId": { - "type" : ["null", "string"], + "type": ["null", "string"], "description": "Id of the related issue.", "readOnly": true }, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_custom_field_contexts.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_custom_field_contexts.json index 6fd0b569f954e..1bada75059281 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_custom_field_contexts.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_custom_field_contexts.json @@ -7,8 +7,8 @@ "description": "The ID of the context." }, "fieldId": { - "type" : ["null", "string"], - "description" : "Id of the related field" + "type": ["null", "string"], + "description": "Id of the related field" }, "name": { "type": "string", diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_properties.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_properties.json index 6f4b18e666ad5..faaa84ba2ec94 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_properties.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_properties.json @@ -7,7 +7,7 @@ "description": "The key of the property. Required on create and update." }, "issueId": { - "type" : ["null", "string"], + "type": ["null", "string"], "description": "Id of the related issue.", "readOnly": true }, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_remote_links.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_remote_links.json index b058f93414953..102a5fba5563d 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_remote_links.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_remote_links.json @@ -7,7 +7,7 @@ "description": "The ID of the link." }, "issueId": { - "type" : ["null", "string"], + "type": ["null", "string"], "description": "Id of the related issue." }, "self": { diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_votes.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_votes.json index 903925b030bcc..913638b8b4f78 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_votes.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_votes.json @@ -8,7 +8,7 @@ "readOnly": true }, "issueId": { - "type" : ["null", "string"], + "type": ["null", "string"], "description": "Id of the related issue.", "readOnly": true }, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_watchers.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_watchers.json index 19c4891777372..069c52a189d30 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_watchers.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_watchers.json @@ -8,7 +8,7 @@ "readOnly": true }, "issueId": { - "type" : ["null", "string"], + "type": ["null", "string"], "description": "Id of the related issue." }, "isWatching": { diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_avatars.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_avatars.json index 2e8b46c62da27..afb993abe8e37 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_avatars.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_avatars.json @@ -7,8 +7,8 @@ "description": "The ID of the avatar." }, "projectId": { - "type" : ["null", "string"], - "description" : "Id of the related project." + "type": ["null", "string"], + "description": "Id of the related project." }, "owner": { "type": "string", diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_permission_schemes.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_permission_schemes.json index 2452d5435b611..ff7c28191eec0 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_permission_schemes.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_permission_schemes.json @@ -7,8 +7,8 @@ "description": "The URL of the issue level security item." }, "projectId": { - "type" : ["null", "string"], - "description" : "Id of the related project." + "type": ["null", "string"], + "description": "Id of the related project." }, "id": { "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tab_fields.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tab_fields.json index 74466e57701d2..0180e3dd6d184 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tab_fields.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tab_fields.json @@ -12,12 +12,12 @@ "description": "The name of the screen tab field. Required on create and update. The maximum length is 255 characters." }, "screenId": { - "type" : ["null", "integer"], - "description" : "Id of the related screen." + "type": ["null", "integer"], + "description": "Id of the related screen." }, "tabId": { - "type" : ["null", "integer"], - "description" : "Id of the related tab." + "type": ["null", "integer"], + "description": "Id of the related tab." } }, "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tabs.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tabs.json index 1806c4b2ff40c..013814e7057da 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tabs.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tabs.json @@ -13,8 +13,8 @@ "description": "The name of the screen tab. The maximum length is 255 characters." }, "screenId": { - "type" : ["null", "integer"], - "description" : "Id of the related screen." + "type": ["null", "integer"], + "description": "Id of the related screen." } }, "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py b/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py index 9f2cf2ec5d485..16fdfb000a4c6 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import logging from datetime import datetime from typing import Any, Dict diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_date_time_transformer.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_date_time_transformer.py index 0bee338a005c0..38d5f334e7767 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_date_time_transformer.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_date_time_transformer.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import pytest from source_jira.source import SourceJira from source_jira.streams import ApplicationRoles diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_migrations/test_config.json b/airbyte-integrations/connectors/source-jira/unit_tests/test_migrations/test_config.json index 52d5f046634ea..8745a889bf96b 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_migrations/test_config.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_migrations/test_config.json @@ -1 +1,10 @@ -{"api_token": "invalid_token", "domain": "invaliddomain.atlassian.net", "email": "no-reply@domain.com", "start_date": "2023-01-01T00:00:00Z", "projects": ["IT1", "IT1", "IT1"], "expand_issue_changelog": true, "render_fields": true, "expand_issue_transition": false} \ No newline at end of file +{ + "api_token": "invalid_token", + "domain": "invaliddomain.atlassian.net", + "email": "no-reply@domain.com", + "start_date": "2023-01-01T00:00:00Z", + "projects": ["IT1", "IT1", "IT1"], + "expand_issue_changelog": true, + "render_fields": true, + "expand_issue_transition": false +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcSavedInfoFetcher.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcSavedInfoFetcher.java index 6056c76778a6f..c469c24f3faae 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcSavedInfoFetcher.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcSavedInfoFetcher.java @@ -9,9 +9,10 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.debezium.CdcSavedInfoFetcher; +import io.airbyte.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; import io.airbyte.integrations.source.relationaldb.models.CdcState; import java.util.Optional; -import io.airbyte.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; + public class MssqlCdcSavedInfoFetcher implements CdcSavedInfoFetcher { private final JsonNode savedOffset; diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java index 55becf5956518..a34e25cbd5cc1 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java @@ -343,10 +343,10 @@ void testReadMultipleTablesIncrementally() throws Exception { database.query(connection -> { connection.fetch(String.format("USE %s;", getDefaultNamespace())); connection.fetch(String.format("CREATE TABLE %s (\n" - + " id int PRIMARY KEY,\n" - + " name VARCHAR(200) NOT NULL,\n" - + " updated_at VARCHAR(200) NOT NULL\n" - + ");", streamOneName)); + + " id int PRIMARY KEY,\n" + + " name VARCHAR(200) NOT NULL,\n" + + " updated_at VARCHAR(200) NOT NULL\n" + + ");", streamOneName)); connection.execute( String.format( "INSERT INTO %s(id, name, updated_at) VALUES (1,'picard', '2004-10-19')", @@ -368,22 +368,22 @@ void testReadMultipleTablesIncrementally() throws Exception { // Insert records into second table database.query(ctx -> { ctx.fetch(String.format("CREATE TABLE %s (\n" - + " id int PRIMARY KEY,\n" - + " name VARCHAR(200) NOT NULL,\n" - + " updated_at DATE NOT NULL\n" - + ");", streamTwoName)); + + " id int PRIMARY KEY,\n" + + " name VARCHAR(200) NOT NULL,\n" + + " updated_at DATE NOT NULL\n" + + ");", streamTwoName)); ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (40,'Jean Luc','2006-10-19')", - streamTwoFullyQualifiedName)); + + "VALUES (40,'Jean Luc','2006-10-19')", + streamTwoFullyQualifiedName)); ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (41, 'Groot', '2006-10-19')", - streamTwoFullyQualifiedName)); + + "VALUES (41, 'Groot', '2006-10-19')", + streamTwoFullyQualifiedName)); ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (42, 'Thanos','2006-10-19')", - streamTwoFullyQualifiedName)); + + "VALUES (42, 'Thanos','2006-10-19')", + streamTwoFullyQualifiedName)); return null; }); // Create records list that we expect to see in the state message @@ -424,7 +424,7 @@ void testReadMultipleTablesIncrementally() throws Exception { // All records in the 2 configured streams should be present assertThat(filterRecords(recordsFromFirstSync)).containsExactlyElementsOf( Stream.concat(getTestMessages(streamOneName).stream().parallel(), - streamTwoExpectedRecords.stream().parallel()).collect(toList())); + streamTwoExpectedRecords.stream().parallel()).collect(toList())); final List actualFirstSyncState = extractStateMessage(messagesFromFirstSync); // Since we are emitting a state message after each record, we should have 1 state for each record - @@ -462,12 +462,12 @@ void testReadMultipleTablesIncrementally() throws Exception { // Extract the incremental states of each stream's first and second state message final List streamOneIncrementalStatesFromFirstSync = List.of(streamOneStateMessagesFromFirstSync.get(0).getStream().getStreamState().get("incremental_state"), - streamOneStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); + streamOneStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); final JsonNode streamOneFinalStreamStateFromFirstSync = streamOneStateMessagesFromFirstSync.get(2).getStream().getStreamState(); final List streamTwoIncrementalStatesFromFirstSync = List.of(streamTwoStateMessagesFromFirstSync.get(0).getStream().getStreamState().get("incremental_state"), - streamTwoStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); + streamTwoStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); final JsonNode streamTwoFinalStreamStateFromFirstSync = streamTwoStateMessagesFromFirstSync.get(2).getStream().getStreamState(); // The incremental_state of each stream's first and second incremental states is expected @@ -484,8 +484,8 @@ void testReadMultipleTablesIncrementally() throws Exception { // switch final List messagesFromSecondSyncWithMixedStates = MoreIterators .toList(source.read(config, configuredCatalog, - Jsons.jsonNode(List.of(streamOneStateMessagesFromFirstSync.get(0), - streamTwoStateMessagesFromFirstSync.get(1))))); + Jsons.jsonNode(List.of(streamOneStateMessagesFromFirstSync.get(0), + streamTwoStateMessagesFromFirstSync.get(1))))); // Extract only state messages for each stream after second sync final List streamOneStateMessagesFromSecondSync = @@ -514,19 +514,19 @@ void testReadMultipleTablesIncrementally() throws Exception { database.query(ctx -> { ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (4,'Hooper','2006-10-19')", - getFullyQualifiedTableName(streamOneName))); + + "VALUES (4,'Hooper','2006-10-19')", + getFullyQualifiedTableName(streamOneName))); ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (43, 'Iron Man', '2006-10-19')", - streamTwoFullyQualifiedName)); + + "VALUES (43, 'Iron Man', '2006-10-19')", + streamTwoFullyQualifiedName)); return null; }); final List messagesFromThirdSync = MoreIterators .toList(source.read(config, configuredCatalog, - Jsons.jsonNode(List.of(streamOneStateMessagesFromSecondSync.get(1), - streamTwoStateMessagesFromSecondSync.get(0))))); + Jsons.jsonNode(List.of(streamOneStateMessagesFromSecondSync.get(1), + streamTwoStateMessagesFromSecondSync.get(0))))); // Extract only state messages, state type, and cursor for each stream after second sync final List streamOneStateMessagesFromThirdSync = @@ -648,32 +648,32 @@ protected List getTestMessages(final String streamName) { return List.of( new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_1, - COL_NAME, "picard", - COL_UPDATED_AT, "2004-10-19")))), + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_1, + COL_NAME, "picard", + COL_UPDATED_AT, "2004-10-19")))), new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_2, - COL_NAME, "crusher", - COL_UPDATED_AT, - "2005-10-19")))), + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_2, + COL_NAME, "crusher", + COL_UPDATED_AT, + "2005-10-19")))), new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_3, - COL_NAME, "vash", - COL_UPDATED_AT, "2006-10-19"))))); + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_3, + COL_NAME, "vash", + COL_UPDATED_AT, "2006-10-19"))))); } private AirbyteStream getAirbyteStream(final String tableName, final String namespace) { return CatalogHelpers.createAirbyteStream( - tableName, - namespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + tableName, + namespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))); } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java index 902ce156a35e3..147c3281a6867 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java @@ -69,10 +69,11 @@ private static Properties commonProperties(final JdbcDatabase database) { if (sourceConfig.get("replication_method").has("server_time_zone")) { final String serverTimeZone = sourceConfig.get("replication_method").get("server_time_zone").asText(); if (!serverTimeZone.isEmpty()) { - /** Per Debezium docs, https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-temporal-types - * this property is now connectionTimeZone - * {@link com.mysql.cj.conf.PropertyKey#connectionTimeZone} - **/ + /** + * Per Debezium docs, + * https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-temporal-types + * this property is now connectionTimeZone {@link com.mysql.cj.conf.PropertyKey#connectionTimeZone} + **/ props.setProperty("database.connectionTimeZone", serverTimeZone); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java index 4fe64b18ee0ee..a25149da6ab18 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java @@ -66,4 +66,5 @@ public AirbyteMessage saveStateAfterCompletionOfSnapshotOfNewStreams() { public boolean compressSchemaHistoryForState() { return COMPRESSION_ENABLED; } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlQueryUtils.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlQueryUtils.java index d6598bd5677e9..3727ddcafad0f 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlQueryUtils.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlQueryUtils.java @@ -146,7 +146,6 @@ public static Map getTableSizeInf return tableSizeInfoMap; } - /** * Iterates through each stream and find the max cursor value and the record count which has that * value based on each cursor field provided by the customer per stream This information is saved in diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cursor_based/MySqlCursorBasedStateManager.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cursor_based/MySqlCursorBasedStateManager.java index 4c53b88452291..c9d656ee3aae2 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cursor_based/MySqlCursorBasedStateManager.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cursor_based/MySqlCursorBasedStateManager.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.source.mysql.cursor_based; import com.google.common.collect.Lists; @@ -78,4 +82,5 @@ private CursorBasedStatus generateDbStreamState(final AirbyteStreamNameNamespace } return state; } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java index a2428ad84edd8..9bdfa3727efcc 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java @@ -218,10 +218,10 @@ private ConfiguredAirbyteCatalog getConfiguredCatalogWithPartialColumns() { .withSyncMode(INCREMENTAL) .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s", STREAM_NAME), - String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), - Field.of("id", JsonSchemaType.NUMBER) - /* no name field */) + String.format("%s", STREAM_NAME), + String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), + Field.of("id", JsonSchemaType.NUMBER) + /* no name field */) .withSourceDefinedCursor(true) .withSourceDefinedPrimaryKey(List.of(List.of("id"))) .withSupportedSyncModes( @@ -230,10 +230,10 @@ private ConfiguredAirbyteCatalog getConfiguredCatalogWithPartialColumns() { .withSyncMode(INCREMENTAL) .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s", STREAM_NAME2), - String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), - /* no name field */ - Field.of("id", JsonSchemaType.NUMBER)) + String.format("%s", STREAM_NAME2), + String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), + /* no name field */ + Field.of("id", JsonSchemaType.NUMBER)) .withSourceDefinedCursor(true) .withSourceDefinedPrimaryKey(List.of(List.of("id"))) .withSupportedSyncModes( diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslCaCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslCaCertificateSourceAcceptanceTest.java index 81e251fce590b..309cfd258e572 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslCaCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslCaCertificateSourceAcceptanceTest.java @@ -55,4 +55,5 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc grantCorrectPermissions(); createAndPopulateTables(); } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslRequiredSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslRequiredSourceAcceptanceTest.java index 011769bc62798..5ae7ebf22fbe7 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslRequiredSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslRequiredSourceAcceptanceTest.java @@ -52,4 +52,5 @@ protected void setupEnvironment(final TestDestinationEnv environment) { private void alterUserRequireSsl() { executeQuery("ALTER USER " + container.getUsername() + " REQUIRE SSL;"); } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlDatatypeAccuracyTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlDatatypeAccuracyTest.java index 4bded16335fde..3a1b979795588 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlDatatypeAccuracyTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlDatatypeAccuracyTest.java @@ -30,6 +30,7 @@ @ExtendWith(SystemStubsExtension.class) public class MySqlDatatypeAccuracyTest extends AbstractMySqlSourceDatatypeTest { + @SystemStub private EnvironmentVariables environmentVariables; diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceDatatypeTest.java index 74bb6bda94165..3e24d7adaa87a 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceDatatypeTest.java @@ -24,6 +24,7 @@ @ExtendWith(SystemStubsExtension.class) public class MySqlSourceDatatypeTest extends AbstractMySqlSourceDatatypeTest { + @SystemStub private EnvironmentVariables environmentVariables; diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java index 21554cffa3f53..9a214a0da6ea6 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java @@ -66,8 +66,8 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Random; import java.util.Properties; +import java.util.Random; import java.util.Set; import java.util.stream.Collectors; import javax.sql.DataSource; @@ -705,9 +705,11 @@ public void testTwoStreamSync() throws Exception { } /** - * This test creates lots of tables increasing the schema history size above the limit of {@link AirbyteSchemaHistoryStorage#SIZE_LIMIT_TO_COMPRESS_MB} - * forcing the {@link AirbyteSchemaHistoryStorage#read()} method to compress the schema history blob as part of the state message - * which allows us to test that the next sync is able to work fine when provided with a compressed blob in the state. + * This test creates lots of tables increasing the schema history size above the limit of + * {@link AirbyteSchemaHistoryStorage#SIZE_LIMIT_TO_COMPRESS_MB} forcing the + * {@link AirbyteSchemaHistoryStorage#read()} method to compress the schema history blob as part of + * the state message which allows us to test that the next sync is able to work fine when provided + * with a compressed blob in the state. */ @Test public void testCompressedSchemaHistory() throws Exception { @@ -785,4 +787,5 @@ private static String generateRandomStringOf32Characters() { return randomString.toString(); } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java index a0ecf1cc1634f..43715985a87e1 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java @@ -90,7 +90,6 @@ class MySqlJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { protected Database database; protected DSLContext dslContext; - @BeforeAll static void init() throws Exception { container = new MySQLContainer<>("mysql:8.0") @@ -107,20 +106,20 @@ static void init() throws Exception { public void setup() throws Exception { environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, Strings.addRandomSuffix("db", "_", 10)) - .put(JdbcUtils.USERNAME_KEY, TEST_USER) - .put(JdbcUtils.PASSWORD_KEY, TEST_PASSWORD.call()) - .build()); + .put(JdbcUtils.HOST_KEY, container.getHost()) + .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .put(JdbcUtils.DATABASE_KEY, Strings.addRandomSuffix("db", "_", 10)) + .put(JdbcUtils.USERNAME_KEY, TEST_USER) + .put(JdbcUtils.PASSWORD_KEY, TEST_PASSWORD.call()) + .build()); dslContext = DSLContextFactory.create( config.get(JdbcUtils.USERNAME_KEY).asText(), config.get(JdbcUtils.PASSWORD_KEY).asText(), DatabaseDriver.MYSQL.getDriverClassName(), String.format("jdbc:mysql://%s:%s", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asText()), + config.get(JdbcUtils.HOST_KEY).asText(), + config.get(JdbcUtils.PORT_KEY).asText()), SQLDialect.MYSQL); database = new Database(dslContext); @@ -173,10 +172,10 @@ void testReadMultipleTablesIncrementally() throws Exception { database.query(connection -> { connection.fetch(String.format("USE %s;", getDefaultNamespace())); connection.fetch(String.format("CREATE TABLE %s (\n" - + " id int PRIMARY KEY,\n" - + " name VARCHAR(200) NOT NULL,\n" - + " updated_at VARCHAR(200) NOT NULL\n" - + ");", streamOneName)); + + " id int PRIMARY KEY,\n" + + " name VARCHAR(200) NOT NULL,\n" + + " updated_at VARCHAR(200) NOT NULL\n" + + ");", streamOneName)); connection.execute( String.format( "INSERT INTO %s(id, name, updated_at) VALUES (1,'picard', '2004-10-19')", @@ -198,22 +197,22 @@ void testReadMultipleTablesIncrementally() throws Exception { // Insert records into second table database.query(ctx -> { ctx.fetch(String.format("CREATE TABLE %s (\n" - + " id int PRIMARY KEY,\n" - + " name VARCHAR(200) NOT NULL,\n" - + " updated_at DATE NOT NULL\n" - + ");", streamTwoName)); + + " id int PRIMARY KEY,\n" + + " name VARCHAR(200) NOT NULL,\n" + + " updated_at DATE NOT NULL\n" + + ");", streamTwoName)); ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (40,'Jean Luc','2006-10-19')", - streamTwoFullyQualifiedName)); + + "VALUES (40,'Jean Luc','2006-10-19')", + streamTwoFullyQualifiedName)); ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (41, 'Groot', '2006-10-19')", - streamTwoFullyQualifiedName)); + + "VALUES (41, 'Groot', '2006-10-19')", + streamTwoFullyQualifiedName)); ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (42, 'Thanos','2006-10-19')", - streamTwoFullyQualifiedName)); + + "VALUES (42, 'Thanos','2006-10-19')", + streamTwoFullyQualifiedName)); return null; }); // Create records list that we expect to see in the state message @@ -254,7 +253,7 @@ void testReadMultipleTablesIncrementally() throws Exception { // All records in the 2 configured streams should be present assertThat(filterRecords(recordsFromFirstSync)).containsExactlyElementsOf( Stream.concat(getTestMessages(streamOneName).stream().parallel(), - streamTwoExpectedRecords.stream().parallel()).collect(toList())); + streamTwoExpectedRecords.stream().parallel()).collect(toList())); final List actualFirstSyncState = extractStateMessage(messagesFromFirstSync); // Since we are emitting a state message after each record, we should have 1 state for each record - @@ -292,12 +291,12 @@ void testReadMultipleTablesIncrementally() throws Exception { // Extract the incremental states of each stream's first and second state message final List streamOneIncrementalStatesFromFirstSync = List.of(streamOneStateMessagesFromFirstSync.get(0).getStream().getStreamState().get("incremental_state"), - streamOneStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); + streamOneStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); final JsonNode streamOneFinalStreamStateFromFirstSync = streamOneStateMessagesFromFirstSync.get(2).getStream().getStreamState(); final List streamTwoIncrementalStatesFromFirstSync = List.of(streamTwoStateMessagesFromFirstSync.get(0).getStream().getStreamState().get("incremental_state"), - streamTwoStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); + streamTwoStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); final JsonNode streamTwoFinalStreamStateFromFirstSync = streamTwoStateMessagesFromFirstSync.get(2).getStream().getStreamState(); // The incremental_state of each stream's first and second incremental states is expected @@ -310,11 +309,12 @@ void testReadMultipleTablesIncrementally() throws Exception { // Sync should work with a primaryKey state AND a cursor-based state from each stream // Forcing a sync with // - stream one state still being the first record read via Primary Key. - // - stream two state being the Primary Key state before the final emitted state before the cursor switch + // - stream two state being the Primary Key state before the final emitted state before the cursor + // switch final List messagesFromSecondSyncWithMixedStates = MoreIterators .toList(source.read(config, configuredCatalog, - Jsons.jsonNode(List.of(streamOneStateMessagesFromFirstSync.get(0), - streamTwoStateMessagesFromFirstSync.get(1))))); + Jsons.jsonNode(List.of(streamOneStateMessagesFromFirstSync.get(0), + streamTwoStateMessagesFromFirstSync.get(1))))); // Extract only state messages for each stream after second sync final List streamOneStateMessagesFromSecondSync = @@ -343,19 +343,19 @@ void testReadMultipleTablesIncrementally() throws Exception { database.query(ctx -> { ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (4,'Hooper','2006-10-19')", - getFullyQualifiedTableName(streamOneName))); + + "VALUES (4,'Hooper','2006-10-19')", + getFullyQualifiedTableName(streamOneName))); ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (43, 'Iron Man', '2006-10-19')", - streamTwoFullyQualifiedName)); + + "VALUES (43, 'Iron Man', '2006-10-19')", + streamTwoFullyQualifiedName)); return null; }); final List messagesFromThirdSync = MoreIterators .toList(source.read(config, configuredCatalog, - Jsons.jsonNode(List.of(streamOneStateMessagesFromSecondSync.get(1), - streamTwoStateMessagesFromSecondSync.get(0))))); + Jsons.jsonNode(List.of(streamOneStateMessagesFromSecondSync.get(1), + streamTwoStateMessagesFromSecondSync.get(0))))); // Extract only state messages, state type, and cursor for each stream after second sync final List streamOneStateMessagesFromThirdSync = @@ -470,17 +470,17 @@ protected DbStreamState buildStreamState(final ConfiguredAirbyteStream configure protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { final List expectedMessages = new ArrayList<>(); expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(ImmutableMap - .of(COL_ID, ID_VALUE_4, - COL_NAME, "riker", - COL_UPDATED_AT, "2006-10-19"))))); + .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_4, + COL_NAME, "riker", + COL_UPDATED_AT, "2006-10-19"))))); expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(ImmutableMap - .of(COL_ID, ID_VALUE_5, - COL_NAME, "data", - COL_UPDATED_AT, "2006-10-19"))))); + .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_5, + COL_NAME, "data", + COL_UPDATED_AT, "2006-10-19"))))); final DbStreamState state = new CursorBasedStatus() .withStateType(StateType.CURSOR_BASED) .withVersion(2L) @@ -508,34 +508,32 @@ protected List getTestMessages(final String streamName) { return List.of( new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_1, - COL_NAME, "picard", - COL_UPDATED_AT, "2004-10-19")))), + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_1, + COL_NAME, "picard", + COL_UPDATED_AT, "2004-10-19")))), new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_2, - COL_NAME, "crusher", - COL_UPDATED_AT, - "2005-10-19")))), + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_2, + COL_NAME, "crusher", + COL_UPDATED_AT, + "2005-10-19")))), new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_3, - COL_NAME, "vash", - COL_UPDATED_AT, "2006-10-19"))))); + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_3, + COL_NAME, "vash", + COL_UPDATED_AT, "2006-10-19"))))); } - - private AirbyteStream getAirbyteStream(final String tableName, final String namespace) { return CatalogHelpers.createAirbyteStream( - tableName, - namespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + tableName, + namespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))); } @@ -544,27 +542,27 @@ private AirbyteStream getAirbyteStream(final String tableName, final String name protected AirbyteCatalog getCatalog(final String defaultNamespace) { return new AirbyteCatalog().withStreams(Lists.newArrayList( CatalogHelpers.createAirbyteStream( - TABLE_NAME, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + TABLE_NAME, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))), CatalogHelpers.createAirbyteStream( - TABLE_NAME_WITHOUT_PK, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + TABLE_NAME_WITHOUT_PK, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(Collections.emptyList()), CatalogHelpers.createAirbyteStream( - TABLE_NAME_COMPOSITE_PK, - defaultNamespace, - Field.of(COL_FIRST_NAME, JsonSchemaType.STRING), - Field.of(COL_LAST_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + TABLE_NAME_COMPOSITE_PK, + defaultNamespace, + Field.of(COL_FIRST_NAME, JsonSchemaType.STRING), + Field.of(COL_LAST_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey( List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); @@ -574,29 +572,29 @@ protected AirbyteCatalog getCatalog(final String defaultNamespace) { @Override protected List createExpectedTestMessages(final List states) { return supportsPerStream() - ? states.stream() - .map(s -> new AirbyteMessage().withType(Type.STATE) - .withState( - new AirbyteStateMessage().withType(AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState() - .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) - .withStreamState(Jsons.jsonNode(s))))) - .collect( - Collectors.toList()) - : List.of(new AirbyteMessage().withType(Type.STATE).withState(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY))); + ? states.stream() + .map(s -> new AirbyteMessage().withType(Type.STATE) + .withState( + new AirbyteStateMessage().withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) + .withStreamState(Jsons.jsonNode(s))))) + .collect( + Collectors.toList()) + : List.of(new AirbyteMessage().withType(Type.STATE).withState(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY))); } @Override protected List createState(final List states) { return supportsPerStream() - ? states.stream() - .map(s -> new AirbyteStateMessage().withType(AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState() - .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) - .withStreamState(Jsons.jsonNode(s)))) - .collect( - Collectors.toList()) - : List.of(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY)); + ? states.stream() + .map(s -> new AirbyteStateMessage().withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) + .withStreamState(Jsons.jsonNode(s)))) + .collect( + Collectors.toList()) + : List.of(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY)); } @Override @@ -608,5 +606,5 @@ protected JsonNode getStateData(final AirbyteMessage airbyteMessage, final Strin throw new IllegalArgumentException("Stream not found in state message: " + streamName); } -} +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json index ee3c4b66de8da..dc44c707ad24c 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json @@ -21,4 +21,4 @@ } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/manifest.yaml b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/manifest.yaml index c1f09efbb0c47..1ae2572cc8c59 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/manifest.yaml +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/manifest.yaml @@ -16,7 +16,7 @@ definitions: requester: type: HttpRequester - url_base: 'https://api-m.{{ "sandbox." if config["is_sandbox"] }}paypal.com/v1/reporting/' + url_base: 'https://api-m.{{ "sandbox." if config["is_sandbox"] }}paypal.com/v1/reporting/' path: "{{ parameters.path }}" http_method: GET request_headers: @@ -24,11 +24,11 @@ definitions: authenticator: type: CustomAuthenticator class_name: source_paypal_transaction.components.PayPalOauth2Authenticator - client_id: '{{ config[''client_id''] }}' - client_secret: '{{ config[''client_secret''] }}' + client_id: "{{ config['client_id'] }}" + client_secret: "{{ config['client_secret'] }}" refresh_request_body: Content-Type: application/x-www-form-urlencoded - token_refresh_endpoint: 'https://api-m.{{ "sandbox." if config["is_sandbox"] }}paypal.com/v1/oauth2/token' + token_refresh_endpoint: 'https://api-m.{{ "sandbox." if config["is_sandbox"] }}paypal.com/v1/oauth2/token' grant_type: client_credentials expires_in_name: expires_in access_token_name: access_token @@ -78,18 +78,18 @@ definitions: fields: - path: - transaction_id - value: '{{ record[''transaction_info''][''transaction_id''] }}' + value: "{{ record['transaction_info']['transaction_id'] }}" incremental_sync: type: DatetimeBasedCursor cursor_field: transaction_updated_date cursor_datetime_formats: - - '%Y-%m-%dT%H:%M:%SZ' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + - "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_datetime: type: MinMaxDatetime datetime: >- {{ max( format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ'), day_delta(-1095, format='%Y-%m-%dT%H:%M:%SZ') ) }} - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption field_name: start_date @@ -100,8 +100,8 @@ definitions: inject_into: request_parameter end_datetime: type: MinMaxDatetime - datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" step: P15D cursor_granularity: PT1S $parameters: @@ -132,18 +132,18 @@ definitions: fields: - path: - as_of_time - value: '{{ format_datetime(record[''as_of_time''], ''%Y-%m-%dT%H:%M:%SZ'') }}' + value: "{{ format_datetime(record['as_of_time'], '%Y-%m-%dT%H:%M:%SZ') }}" incremental_sync: type: DatetimeBasedCursor cursor_field: as_of_time cursor_datetime_formats: - - '%Y-%m-%dT%H:%M:%SZ' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + - "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_datetime: type: MinMaxDatetime datetime: >- {{ max( format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ'), day_delta(-1095, format='%Y-%m-%dT%H:%M:%SZ') ) }} - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption field_name: as_of_time @@ -201,4 +201,4 @@ spec: type: "string" title: "Refresh token" description: "The key to refresh the expired access token." - airbyte_secret: true \ No newline at end of file + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json index afa3150857ddb..58031b32fb5e5 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json @@ -4,7 +4,7 @@ "stream": { "name": "deals", "json_schema": {}, - "supported_sync_modes": ["full_refresh","incremental"], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["update_time"] }, diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml index 42240785560be..319bcde916bc6 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml @@ -12,12 +12,12 @@ definitions: extractor: type: DpathExtractor field_path: ["data", "*", "data"] - + selector_users: type: RecordSelector extractor: type: DpathExtractor - field_path: ["data", "*", "data","*"] + field_path: ["data", "*", "data", "*"] requester: type: HttpRequester @@ -107,7 +107,7 @@ definitions: path: "/recents" lead_labels_stream: - # Does not support pagination + # Does not support pagination $ref: "#/definitions/base_stream" name: "lead_labels" retriever: @@ -148,7 +148,7 @@ definitions: path: "/recents" activity_types_stream: -# This stream didnt have limit as query_parameter + # This stream didnt have limit as query_parameter $ref: "#/definitions/base_stream" name: "activity_types" retriever: @@ -302,7 +302,6 @@ definitions: parent_key: "id" partition_field: "parent_id" - streams: - "#/definitions/deals_stream" - "#/definitions/deal_fields_stream" @@ -351,9 +350,10 @@ spec: airbyte_secret: true replication_start_date: title: Start Date - description: UTC date and time in the format 2017-01-25T00:00:00Z. Any data + description: + UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. When specified and not None, then stream will behave as incremental examples: - - '2017-01-25 00:00:00Z' + - "2017-01-25 00:00:00Z" type: string diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json index 997124c2b50f2..2901d9c336f42 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json @@ -57,7 +57,7 @@ "type": ["null", "boolean"] }, "json_column_flag": { - "type": ["null", "boolean"] + "type": ["null", "boolean"] }, "parent_id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deal_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deal_fields.json index 4b06133de6012..805265c218640 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deal_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deal_fields.json @@ -102,7 +102,7 @@ } }, "mandatory_flag": { - "type": ["null", "object","boolean","integer"] + "type": ["null", "object", "boolean", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json index f24cb9d06a698..dff11aa1ddba7 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json @@ -59,19 +59,19 @@ "type": ["null", "boolean"] }, "parent_id": { - "type": ["null", "integer"] + "type": ["null", "integer"] }, "id_suffix": { - "type": ["null", "string"] + "type": ["null", "string"] }, "is_subfield": { - "type": ["null", "boolean"] + "type": ["null", "boolean"] }, "json_column_flag": { - "type": ["null", "boolean"] + "type": ["null", "boolean"] }, "group_id": { - "type": ["null", "integer"] + "type": ["null", "integer"] }, "options": { "type": ["null", "array"], diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json index 858901f328f1a..62d35b1099b01 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json @@ -198,10 +198,10 @@ "type": ["null", "string"] }, "category_id": { - "type": ["null", "integer"] + "type": ["null", "integer"] }, "delete_time": { - "type": ["null", "string"] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java index a524860752b55..706dca1a885e5 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java @@ -321,8 +321,8 @@ public static List filterStreamsUnderVacuumForCtidSync( } public static Map getTableMaxTupleForStreams(final JdbcDatabase database, - final List streams, - final String quoteString) { + final List streams, + final String quoteString) { final Map tableMaxTupleEstimates = new HashMap<>(); streams.forEach(stream -> { final AirbyteStreamNameNamespacePair namespacePair = @@ -334,8 +334,8 @@ public static Map getTableMaxTupleForSt } public static int getTableMaxTupleForStream(final JdbcDatabase database, - final AirbyteStreamNameNamespacePair stream, - final String quoteString) { + final AirbyteStreamNameNamespacePair stream, + final String quoteString) { try { final String streamName = stream.getName(); final String schemaName = stream.getNamespace(); diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index 56c6a20656d83..7c365a7373acf 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -494,8 +494,8 @@ public List> getIncrementalIterators(final getQuoteString()); final Map tablesMaxTuple = - CtidUtils.isTidRangeScanCapableDBServer(database) ? null : - PostgresQueryUtils.getTableMaxTupleForStreams(database, finalListOfStreamsToBeSyncedViaCtid, getQuoteString()); + CtidUtils.isTidRangeScanCapableDBServer(database) ? null + : PostgresQueryUtils.getTableMaxTupleForStreams(database, finalListOfStreamsToBeSyncedViaCtid, getQuoteString()); if (!streamsCategorised.ctidStreams().streamsForCtidSync().isEmpty()) { LOGGER.info("Streams to be synced via ctid : {}", finalListOfStreamsToBeSyncedViaCtid.size()); LOGGER.info("Streams: {}", prettyPrintConfiguredAirbyteStreamList(finalListOfStreamsToBeSyncedViaCtid)); @@ -563,8 +563,8 @@ public List> getIncrementalIterators(final getQuoteString()); final Map tablesMaxTuple = - CtidUtils.isTidRangeScanCapableDBServer(database) ? null : - PostgresQueryUtils.getTableMaxTupleForStreams(database, finalListOfStreamsToBeSyncedViaCtid, getQuoteString()); + CtidUtils.isTidRangeScanCapableDBServer(database) ? null + : PostgresQueryUtils.getTableMaxTupleForStreams(database, finalListOfStreamsToBeSyncedViaCtid, getQuoteString()); if (finalListOfStreamsToBeSyncedViaCtid.isEmpty()) { LOGGER.info("No Streams will be synced via ctid."); diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java index a153d08e8dc4c..17594f4147113 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java @@ -143,8 +143,8 @@ public static List> cdcCtidIteratorsCombin quoteString); final Map tablesMaxTuple = - CtidUtils.isTidRangeScanCapableDBServer(database) ? null : - PostgresQueryUtils.getTableMaxTupleForStreams(database, finalListOfStreamsToBeSyncedViaCtid, quoteString); + CtidUtils.isTidRangeScanCapableDBServer(database) ? null + : PostgresQueryUtils.getTableMaxTupleForStreams(database, finalListOfStreamsToBeSyncedViaCtid, quoteString); final PostgresCtidHandler ctidHandler = new PostgresCtidHandler(sourceConfig, database, ctidPostgresSourceOperations, diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcSavedInfoFetcher.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcSavedInfoFetcher.java index 89dc92149a96f..25f27d8179484 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcSavedInfoFetcher.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcSavedInfoFetcher.java @@ -6,8 +6,8 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.debezium.CdcSavedInfoFetcher; -import io.airbyte.integrations.source.relationaldb.models.CdcState; import io.airbyte.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; +import io.airbyte.integrations.source.relationaldb.models.CdcState; import java.util.Optional; public class PostgresCdcSavedInfoFetcher implements CdcSavedInfoFetcher { diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/Ctid.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/Ctid.java index e3284c0ae0829..bd6b4ae9bb4a4 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/Ctid.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/Ctid.java @@ -75,4 +75,5 @@ public int hashCode() { public static Ctid inc(final Ctid ctid, final long maxTuple) { return (ctid.tuple + 1 > maxTuple) ? Ctid.of(ctid.page + 1, 1) : Ctid.of(ctid.page, ctid.tuple + 1); } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidUtils.java index 0f26c218817e8..68228b06ae909 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidUtils.java @@ -12,7 +12,6 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.SyncMode; -import java.sql.SQLException; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -61,8 +60,9 @@ public record StreamsCategorised (CtidStreams ctidStreams, } /** - * Postgres servers version 14 and above are capable of running a tid range scan. - * Used by ctid queries + * Postgres servers version 14 and above are capable of running a tid range scan. Used by ctid + * queries + * * @param database database * @return true for Tid scan capable server */ diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIterator.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIterator.java index 1aa7d3039d291..ef490c34cfad4 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIterator.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIterator.java @@ -66,6 +66,7 @@ public class InitialSyncCtidIterator extends AbstractIterator i private int numberOfTimesReSynced = 0; private boolean subQueriesInitialized = false; private final boolean tidRangeScanCapableDBServer; + public InitialSyncCtidIterator(final CtidStateManager ctidStateManager, final JdbcDatabase database, final CtidPostgresSourceOperations sourceOperations, diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractCdcPostgresSourceSslAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractCdcPostgresSourceSslAcceptanceTest.java index b3a912a52a6a8..0441cd9cdb20b 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractCdcPostgresSourceSslAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractCdcPostgresSourceSslAcceptanceTest.java @@ -78,6 +78,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc } protected abstract String getServerImageName(); + public abstract ImmutableMap getCertificateConfiguration(); } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceLegacyCtidTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceLegacyCtidTest.java index a5a3c6146d699..44299c3e8881b 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceLegacyCtidTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceLegacyCtidTest.java @@ -1,8 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.io.airbyte.integration_tests.sources; -public class CDCPostgresSourceCaCertificateSslAcceptanceLegacyCtidTest extends CDCPostgresSourceCaCertificateSslAcceptanceTest{ +public class CDCPostgresSourceCaCertificateSslAcceptanceLegacyCtidTest extends CDCPostgresSourceCaCertificateSslAcceptanceTest { + @Override protected String getServerImageName() { return "postgres:12-bullseye"; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceTest.java index 0936751640fd7..1e052877768ca 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceTest.java @@ -21,5 +21,4 @@ protected String getServerImageName() { return "postgres:15-bullseye"; } - } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceFullCertificateSslAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceFullCertificateSslAcceptanceTest.java index cd083c1ba8833..20b264e3a0cfd 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceFullCertificateSslAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceFullCertificateSslAcceptanceTest.java @@ -23,4 +23,5 @@ public ImmutableMap getCertificateConfiguration() { protected String getServerImageName() { return "postgres:15-bullseye"; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceLegacyCtidTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceLegacyCtidTest.java index 42b8312bb28c1..7671d7d29d018 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceLegacyCtidTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceLegacyCtidTest.java @@ -1,8 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.io.airbyte.integration_tests.sources; -public class CdcPostgresSourceAcceptanceLegacyCtidTest extends CdcPostgresSourceAcceptanceTest{ +public class CdcPostgresSourceAcceptanceLegacyCtidTest extends CdcPostgresSourceAcceptanceTest { + @Override protected String getServerImageName() { return "postgres:13-alpine"; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java index f96cb2c49ceec..8583922b36f58 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java @@ -225,4 +225,5 @@ private void verifyFieldNotExist(final List records, final protected String getServerImageName() { return "postgres:15-alpine"; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceLegacyCtidTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceLegacyCtidTest.java index 58bb28ab99cbf..f0fce358a3c00 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceLegacyCtidTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceLegacyCtidTest.java @@ -1,9 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.io.airbyte.integration_tests.sources; -public class PostgresSourceAcceptanceLegacyCtidTest extends PostgresSourceAcceptanceTest{ +public class PostgresSourceAcceptanceLegacyCtidTest extends PostgresSourceAcceptanceTest { @Override protected String getServerImageName() { return "postgres:13-alpine"; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceLegacyCtidTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceLegacyCtidTest.java index 6c284994bdecf..2461331a5ca14 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceLegacyCtidTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceLegacyCtidTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.source.postgres; public class CdcPostgresSourceLegacyCtidTest extends CdcPostgresSourceTest { @@ -6,4 +10,5 @@ public class CdcPostgresSourceLegacyCtidTest extends CdcPostgresSourceTest { protected String getServerImageName() { return "debezium/postgres:13-alpine"; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java index fefdb69b29f9e..a036cde99d2f2 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java @@ -993,11 +993,16 @@ protected void ctidIteratorPageSizeTest() throws Exception { } @Override - protected void compareTargetPositionFromTheRecordsWithTargetPostionGeneratedBeforeSync(final CdcTargetPosition targetPosition, final AirbyteRecordMessage record) { - // The LSN from records should be either equal or grater than the position value before the sync started. - // The current Write-Ahead Log (WAL) position can move ahead even without any data modifications (INSERT, UPDATE, DELETE) - // The start and end of transactions, even read-only ones, are recorded in the WAL. So, simply starting and committing a transaction can cause the WAL location to move forward. - // Periodic checkpoints, which write dirty pages from memory to disk to ensure database consistency, generate WAL records. Checkpoints happen even if there are no active data modifications + protected void compareTargetPositionFromTheRecordsWithTargetPostionGeneratedBeforeSync(final CdcTargetPosition targetPosition, + final AirbyteRecordMessage record) { + // The LSN from records should be either equal or grater than the position value before the sync + // started. + // The current Write-Ahead Log (WAL) position can move ahead even without any data modifications + // (INSERT, UPDATE, DELETE) + // The start and end of transactions, even read-only ones, are recorded in the WAL. So, simply + // starting and committing a transaction can cause the WAL location to move forward. + // Periodic checkpoints, which write dirty pages from memory to disk to ensure database consistency, + // generate WAL records. Checkpoints happen even if there are no active data modifications assert targetPosition instanceof PostgresCdcTargetPosition; assertTrue(extractPosition(record.getData()).targetLsn.compareTo(((PostgresCdcTargetPosition) targetPosition).targetLsn) >= 0); } @@ -1005,4 +1010,5 @@ protected void compareTargetPositionFromTheRecordsWithTargetPostionGeneratedBefo protected String getServerImageName() { return "debezium/postgres:15-alpine"; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIteratorTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIteratorTest.java index a6f897cf4dd77..03b23fdf4b03b 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIteratorTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIteratorTest.java @@ -50,8 +50,8 @@ void testCtidQueryBounds() { Pair.of(Ctid.ZERO, null)); assertEquals(expected, chunks); - chunks = InitialSyncCtidIterator.ctidQueryPlan(Ctid.of(8,1), 819200L, 81920L, 50, EIGHT_KB); - expected = List.of(Pair.of(Ctid.of(8,1), Ctid.of(9,0)), Pair.of(Ctid.of(9,0), Ctid.of(10,0))); + chunks = InitialSyncCtidIterator.ctidQueryPlan(Ctid.of(8, 1), 819200L, 81920L, 50, EIGHT_KB); + expected = List.of(Pair.of(Ctid.of(8, 1), Ctid.of(9, 0)), Pair.of(Ctid.of(9, 0), Ctid.of(10, 0))); assertEquals(expected, chunks); } @@ -59,58 +59,58 @@ void testCtidQueryBounds() { void testLegacyCtidQueryBounds() { var chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.of(0, 184), 3805450321L, 8192L, 50, GIGABYTE, 185); var expected = List.of( - Pair.of(Ctid.of (0,185), Ctid.of(27027,185)), - Pair.of(Ctid.of(27028,1), Ctid.of(54054,185)), - Pair.of(Ctid.of(54055,1), Ctid.of(81081,185)), - Pair.of(Ctid.of(81082,1), Ctid.of(108108,185)), - Pair.of(Ctid.of(108109,1), Ctid.of(135135,185)), - Pair.of(Ctid.of(135136,1), Ctid.of(162162,185)), - Pair.of(Ctid.of(162163,1), Ctid.of(189189,185)), - Pair.of(Ctid.of(189190,1), Ctid.of(216216,185)), - Pair.of(Ctid.of(216217,1), Ctid.of(243243,185)), - Pair.of(Ctid.of(243244,1), Ctid.of(270270,185)), - Pair.of(Ctid.of(270271,1), Ctid.of(297297,185)), - Pair.of(Ctid.of(297298,1), Ctid.of(324324,185)), - Pair.of(Ctid.of(324325,1), Ctid.of(351351,185)), - Pair.of(Ctid.of(351352,1), Ctid.of(378378,185)), - Pair.of(Ctid.of(378379,1), Ctid.of(405405,185)), - Pair.of(Ctid.of(405406,1), Ctid.of(432432,185)), - Pair.of(Ctid.of(432433,1), Ctid.of(459459,185)), - Pair.of(Ctid.of(459460,1), Ctid.of(486486,185)), - Pair.of(Ctid.of(486487,1), Ctid.of(513513,185))); + Pair.of(Ctid.of(0, 185), Ctid.of(27027, 185)), + Pair.of(Ctid.of(27028, 1), Ctid.of(54054, 185)), + Pair.of(Ctid.of(54055, 1), Ctid.of(81081, 185)), + Pair.of(Ctid.of(81082, 1), Ctid.of(108108, 185)), + Pair.of(Ctid.of(108109, 1), Ctid.of(135135, 185)), + Pair.of(Ctid.of(135136, 1), Ctid.of(162162, 185)), + Pair.of(Ctid.of(162163, 1), Ctid.of(189189, 185)), + Pair.of(Ctid.of(189190, 1), Ctid.of(216216, 185)), + Pair.of(Ctid.of(216217, 1), Ctid.of(243243, 185)), + Pair.of(Ctid.of(243244, 1), Ctid.of(270270, 185)), + Pair.of(Ctid.of(270271, 1), Ctid.of(297297, 185)), + Pair.of(Ctid.of(297298, 1), Ctid.of(324324, 185)), + Pair.of(Ctid.of(324325, 1), Ctid.of(351351, 185)), + Pair.of(Ctid.of(351352, 1), Ctid.of(378378, 185)), + Pair.of(Ctid.of(378379, 1), Ctid.of(405405, 185)), + Pair.of(Ctid.of(405406, 1), Ctid.of(432432, 185)), + Pair.of(Ctid.of(432433, 1), Ctid.of(459459, 185)), + Pair.of(Ctid.of(459460, 1), Ctid.of(486486, 185)), + Pair.of(Ctid.of(486487, 1), Ctid.of(513513, 185))); assertEquals(expected, chunks); chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(new Ctid("(1314328,123)"), 9805450321L, 8192L, 45, GIGABYTE, 10005); expected = List.of( - Pair.of(Ctid.of("(1314328,124)"), Ctid.of(1314827,10005)), - Pair.of(Ctid.of("(1314828,1)"), Ctid.of(1315326,10005)), - Pair.of(Ctid.of("(1315327,1)"), Ctid.of(1315825,10005)), - Pair.of(Ctid.of("(1315826,1)"), Ctid.of(1316324,10005)), - Pair.of(Ctid.of("(1316325,1)"), Ctid.of(1316823,10005))); + Pair.of(Ctid.of("(1314328,124)"), Ctid.of(1314827, 10005)), + Pair.of(Ctid.of("(1314828,1)"), Ctid.of(1315326, 10005)), + Pair.of(Ctid.of("(1315327,1)"), Ctid.of(1315825, 10005)), + Pair.of(Ctid.of("(1315826,1)"), Ctid.of(1316324, 10005)), + Pair.of(Ctid.of("(1316325,1)"), Ctid.of(1316823, 10005))); assertEquals(expected, chunks); chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.of(0, 0), 380545L, 8192L, 45, GIGABYTE, 55); - expected = List.of(Pair.of(Ctid.of(0,1), Ctid.of(90909,55))); + expected = List.of(Pair.of(Ctid.of(0, 1), Ctid.of(90909, 55))); assertEquals(expected, chunks); chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.of(9876, 5432), 380545L, 8192L, 45, GIGABYTE, 5); expected = List.of( - Pair.of(Ctid.of(9877, 1), Ctid.of(1009877,5))); + Pair.of(Ctid.of(9877, 1), Ctid.of(1009877, 5))); assertEquals(expected, chunks); chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.ZERO, 4096L, 8192L, 45, GIGABYTE, 226); expected = List.of( - Pair.of(Ctid.of(0,1), Ctid.of(22123,226))); + Pair.of(Ctid.of(0, 1), Ctid.of(22123, 226))); assertEquals(expected, chunks); // Simulate an empty table - expected to generate an empty query plan chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.ZERO, 4096L, 8192L, 45, GIGABYTE, 1); - expected = List.of(Pair.of(Ctid.of(0,1), Ctid.of(5000000,1))); + expected = List.of(Pair.of(Ctid.of(0, 1), Ctid.of(5000000, 1))); assertEquals(expected, chunks); - chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.of(8,1), 819200L, 81920L, 50, EIGHT_KB, 1); - expected = List.of(Pair.of(Ctid.of(9,1), Ctid.of(10,1)), Pair.of(Ctid.of(11,1), Ctid.of(11,1))); + chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.of(8, 1), 819200L, 81920L, 50, EIGHT_KB, 1); + expected = List.of(Pair.of(Ctid.of(9, 1), Ctid.of(10, 1)), Pair.of(Ctid.of(11, 1), Ctid.of(11, 1))); assertEquals(expected, chunks); } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/events.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/events.json index 01308d4189424..219fe41cace14 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/events.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/events.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "type": {"type" : ["null", "string"]}, + "type": { "type": ["null", "string"] }, "eventID": { "type": "string" }, @@ -72,95 +72,129 @@ }, "_meta": { "type": ["null", "object"], - "properties" : { - "context": {"type" : ["null", "string"]}, - "contexts": {"type" : ["null", "string"]}, - "entries": {"type" : ["null", "object"]}, - "message": {"type" : ["null", "string"]}, - "packages": {"type" : ["null", "string"]}, - "sdk": {"type" : ["null", "string"]}, - "tags": {"type" : ["null", "object"]}, - "user": {"type" : ["null", "string"]} + "properties": { + "context": { "type": ["null", "string"] }, + "contexts": { "type": ["null", "string"] }, + "entries": { "type": ["null", "object"] }, + "message": { "type": ["null", "string"] }, + "packages": { "type": ["null", "string"] }, + "sdk": { "type": ["null", "string"] }, + "tags": { "type": ["null", "object"] }, + "user": { "type": ["null", "string"] } } }, "context": { - "type" : ["null", "object"], - "properties" : { + "type": ["null", "object"], + "properties": { "emptyList": { - "type" : ["null", "array"] + "type": ["null", "array"] }, "emptyMap": { - "type" : ["null", "object"] + "type": ["null", "object"] }, - "length": {"type" : ["null", "integer"]}, - "results": {"type" : ["null", "array"], "items" : {"type" : "integer"}}, - "session": {"type" : ["null", "object"], "properties" : { - "foo": { - "type" : ["null", "string"] + "length": { "type": ["null", "integer"] }, + "results": { + "type": ["null", "array"], + "items": { "type": "integer" } + }, + "session": { + "type": ["null", "object"], + "properties": { + "foo": { + "type": ["null", "string"] + } } - }}, - "unauthorized": {"type" : ["null", "boolean"]}, - "url": {"type" : ["null", "string"]} + }, + "unauthorized": { "type": ["null", "boolean"] }, + "url": { "type": ["null", "string"] } } }, "contexts": { - "type" : ["null", "object"], - "properties" : { - "browser": {"type" : ["null", "object"], "properties" : { - "name": {"type" : ["null", "string"]}, - "type": {"type" : ["null", "string"]}, - "version": {"type" : ["null", "string"]} - }}, - "client_os": {"type" : ["null", "object"], "properties" : { - "name": {"type" : ["null", "string"]}, - "type": {"type" : ["null", "string"]}, - "version": {"type" : ["null", "string"]} - }} + "type": ["null", "object"], + "properties": { + "browser": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] }, + "version": { "type": ["null", "string"] } + } + }, + "client_os": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] }, + "version": { "type": ["null", "string"] } + } + } + } + }, + "crashFile": { "type": ["null", "string"] }, + "culprit": { "type": ["null", "string"] }, + "dateReceived": { "type": ["null", "string"], "format": "date-time" }, + "dist": { "type": ["null", "string"] }, + "entries": { + "type": ["null", "array"], + "items": { "type": ["null", "object"] }, + "properties": { + "data": { + "type": ["null", "object"], + "properties": { + "formatted": { "type": ["null", "string"] }, + "frames": { + "type": ["null", "object"], + "properties": { + "absPath": { "type": ["null", "string"] }, + "colNo": { "type": ["null", "string"] }, + "context": { "type": ["null", "array"] }, + "filename": { "type": ["null", "string"] }, + "function": { "type": ["null", "string"] }, + "inApp": { "type": ["null", "boolean"] }, + "lineNo": { "type": ["null", "integer"] }, + "lock": { "type": ["null", "integer"] }, + "module": { "type": ["null", "string"] }, + "package": { "type": ["null", "string"] }, + "platform": { "type": ["null", "string"] }, + "rawFunction": { "type": ["null", "string"] }, + "sourceLink": { "type": ["null", "string"] }, + "symbol": { "type": ["null", "string"] }, + "symbolAddr": { "type": ["null", "string"] }, + "trust": { "type": ["null", "string"] } + } + } + } + }, + "type": { "type": ["null", "string"] } + } + }, + "errors": { + "type": ["null", "array"], + "items": { "type": ["null", "string"] } + }, + "fingerprints": { + "type": ["null", "array"], + "items": { "type": ["null", "string"] } + }, + "groupingConfig": { + "type": ["null", "object"], + "properties": { + "enhancements": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] } + } + }, + "location": { "type": ["null", "string"] }, + "metadata": { + "type": ["null", "object"], + "properties": { + "in_app_frame_mix": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] } } }, - "crashFile": {"type" : ["null", "string"]}, - "culprit": {"type" : ["null", "string"]}, - "dateReceived": {"type" : ["null", "string"], "format" : "date-time"}, - "dist": {"type" : ["null", "string"]}, - "entries": {"type" : ["null", "array"], "items" : {"type" : ["null", "object"]}, "properties" : { - "data": {"type" : ["null", "object"], "properties" : { - "formatted": {"type" : ["null", "string"]}, - "frames": {"type" : ["null", "object"], "properties" : { - "absPath": {"type" : ["null", "string"]}, - "colNo": {"type" : ["null", "string"]}, - "context": {"type" : ["null", "array"]}, - "filename": {"type" : ["null", "string"]}, - "function": {"type" : ["null", "string"]}, - "inApp": {"type" : ["null", "boolean"]}, - "lineNo": {"type" : ["null", "integer"]}, - "lock": {"type" : ["null", "integer"]}, - "module": {"type" : ["null", "string"]}, - "package": {"type" : ["null", "string"]}, - "platform": {"type" : ["null", "string"]}, - "rawFunction": {"type" : ["null", "string"]}, - "sourceLink": {"type" : ["null", "string"]}, - "symbol": {"type" : ["null", "string"]}, - "symbolAddr": {"type" : ["null", "string"]}, - "trust": {"type" : ["null", "string"]} - }} - }}, - "type": {"type" : ["null", "string"]} - }}, - "errors": {"type" : ["null", "array"], "items" : {"type" : ["null", "string"]}}, - "fingerprints": {"type" : ["null", "array"], "items" : {"type" : ["null", "string"]}}, - "groupingConfig": {"type" : ["null", "object"], "properties" : { - "enhancements": {"type" : ["null", "string"]}, - "id": {"type" : ["null", "string"]} - }}, - "location": {"type" : ["null", "string"]}, - "metadata": {"type" : ["null", "object"], "properties" : { - "in_app_frame_mix": {"type" : ["null", "string"]}, - "title": {"type" : ["null", "string"]} - }}, - "occurrence": {"type" : ["null", "string"]}, - "packages": {"type" : ["null", "object"]}, - "projectID": {"type" : ["null", "string"]}, - "sdk": {"type" : ["null", "string"]}, - "size": {"type" : ["null", "integer"]} + "occurrence": { "type": ["null", "string"] }, + "packages": { "type": ["null", "object"] }, + "projectID": { "type": ["null", "string"] }, + "sdk": { "type": ["null", "string"] }, + "size": { "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/issues.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/issues.json index 53957231edbc5..c5660d0dc62c0 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/issues.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/issues.json @@ -130,19 +130,19 @@ "type": "integer" }, "isUnhandled": { - "type" : ["null", "boolean"] + "type": ["null", "boolean"] }, "issueCategory": { - "type" : ["null", "string"] + "type": ["null", "string"] }, "issueType": { - "type" : ["null", "string"] + "type": ["null", "string"] }, "platform": { - "type" : ["null", "string"] + "type": ["null", "string"] }, "substatus": { - "type" : ["null", "string"] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json index 061aa7221bdad..a78ca92f0f5fe 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json @@ -399,49 +399,49 @@ "type": "boolean" }, "access": { - "type" : ["null", "array"], - "items" : { - "type" : ["null", "string"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] } }, "builtinSymbolSources": { - "type" : ["null", "array"], - "items" : { - "type" : ["null", "string"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] } }, "dynamicSamplingBiases": { - "type" : ["null", "array"], - "items" : { - "type" : ["null", "object"], - "properties" : { - "id": {"type" : ["null", "string"]}, - "active": {"type" : ["null", "boolean"]} + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] }, + "active": { "type": ["null", "boolean"] } } } }, "eventProcessing": { - "type" : ["null", "object"], - "properties" : { + "type": ["null", "object"], + "properties": { "symbolicationDegraded": { - "type" : ["null", "boolean"] + "type": ["null", "boolean"] } } }, - "fingerprintingRules": {"type" : ["null", "string"]}, - "firstTransactionEvent": {"type" : ["null", "boolean"]}, - "groupingAutoUpdate": {"type" : ["null", "boolean"]}, - "groupingConfig": {"type" : ["null", "string"]}, - "groupingEnhancements": {"type" : ["null", "string"]}, - "groupingEnhancementsBase": {"type" : ["null", "string"]}, - "hasMinifiedStackTrace": {"type" : ["null", "boolean"]}, - "hasMonitors": {"type" : ["null", "boolean"]}, - "hasProfiles": {"type" : ["null", "boolean"]}, - "hasReplays": {"type" : ["null", "boolean"]}, - "hasSessions": {"type" : ["null", "boolean"]}, - "recapServerUrl": {"type" : ["null", "string"]}, - "secondaryGroupingConfig": {"type" : ["null", "string"]}, - "secondaryGroupingExpiry": {"type" : ["null", "integer"]}, - "symbolSources": {"type" : ["null", "string"]} + "fingerprintingRules": { "type": ["null", "string"] }, + "firstTransactionEvent": { "type": ["null", "boolean"] }, + "groupingAutoUpdate": { "type": ["null", "boolean"] }, + "groupingConfig": { "type": ["null", "string"] }, + "groupingEnhancements": { "type": ["null", "string"] }, + "groupingEnhancementsBase": { "type": ["null", "string"] }, + "hasMinifiedStackTrace": { "type": ["null", "boolean"] }, + "hasMonitors": { "type": ["null", "boolean"] }, + "hasProfiles": { "type": ["null", "boolean"] }, + "hasReplays": { "type": ["null", "boolean"] }, + "hasSessions": { "type": ["null", "boolean"] }, + "recapServerUrl": { "type": ["null", "string"] }, + "secondaryGroupingConfig": { "type": ["null", "string"] }, + "secondaryGroupingExpiry": { "type": ["null", "integer"] }, + "symbolSources": { "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/projects.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/projects.json index 62a2a56ea03c5..2883aad1db392 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/projects.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/projects.json @@ -116,16 +116,16 @@ "enum": ["active", "disabled", "pending_deletion", "deletion_in_progress"] }, "access": { - "type" : ["null", "array"], - "items" : { - "type" : ["null", "string"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] } }, - "firstTransactionEvent": {"type" : ["null", "boolean"]}, - "hasMonitors": {"type" : ["null", "boolean"]}, - "hasProfiles": {"type" : ["null", "boolean"]}, - "hasReplays": {"type" : ["null", "boolean"]}, - "hasSessions": {"type" : ["null", "boolean"]}, - "hasMinifiedStackTrace": {"type" : ["null", "boolean"]} + "firstTransactionEvent": { "type": ["null", "boolean"] }, + "hasMonitors": { "type": ["null", "boolean"] }, + "hasProfiles": { "type": ["null", "boolean"] }, + "hasReplays": { "type": ["null", "boolean"] }, + "hasSessions": { "type": ["null", "boolean"] }, + "hasMinifiedStackTrace": { "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json index c581d5430c72a..d6ea85853ac01 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json @@ -85,24 +85,26 @@ "type": ["null", "string"] }, "currentProjectMeta": { - "type" : ["null", "object"] - }, - "status": {"type" : ["null", "string"]}, - "userAgent": {"type" : ["null", "string"]}, - "versionInfo": {"type" : ["null", "object"], - "properties" : { - "buildHash": {"type" : ["null", "string"]}, - "description": {"type" : ["null", "string"]}, - "package": {"type" : ["null", "string"]}, - "version": {"type" : ["null", "object"], - "properties" : { - "buildCode": {"type" : ["null", "string"]}, - "components": {"type" : ["null", "integer"]}, - "major": {"type" : ["null", "integer"]}, - "minor": {"type" : ["null", "integer"]}, - "patch": {"type" : ["null", "integer"]}, - "pre": {"type" : ["null", "string"]}, - "raw": {"type" : ["null", "string"]} + "type": ["null", "object"] + }, + "status": { "type": ["null", "string"] }, + "userAgent": { "type": ["null", "string"] }, + "versionInfo": { + "type": ["null", "object"], + "properties": { + "buildHash": { "type": ["null", "string"] }, + "description": { "type": ["null", "string"] }, + "package": { "type": ["null", "string"] }, + "version": { + "type": ["null", "object"], + "properties": { + "buildCode": { "type": ["null", "string"] }, + "components": { "type": ["null", "integer"] }, + "major": { "type": ["null", "integer"] }, + "minor": { "type": ["null", "integer"] }, + "patch": { "type": ["null", "integer"] }, + "pre": { "type": ["null", "string"] }, + "raw": { "type": ["null", "string"] } } } } diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json index bc5819d2c48ac..29376d5432ca1 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json @@ -1,601 +1,601 @@ [ - { - "type": "STREAM", - "stream": { - "stream_state": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "customers" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "orders" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "draft_orders" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "products" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "updatedAt": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "products_graph_ql" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "abandoned_checkouts" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafields" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "id": 99999999999999 - }, - "stream_descriptor": { - "name": "collects" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "custom_collections" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "orders": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "created_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "order_refunds" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "orders": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "id": 99999999999999 - }, - "stream_descriptor": { - "name": "order_risks" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "orders": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "created_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "transactions" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "processed_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "tender_transactions" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "pages" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "price_rules" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "price_rules": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "discount_codes" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "products": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "inventory_items" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "locations": {}, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "inventory_levels" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "orders": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "id": 99999999999999 - }, - "stream_descriptor": { - "name": "fulfillment_orders" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "orders": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "fulfillments" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": {}, - "stream_descriptor": { - "name": "balance_transactions" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "id": 99999999999999, - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "stream_descriptor": { - "name": "articles" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "articles": { - "id": 99999999999999, - "deleted": {} - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_articles" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "id": 99999999999999, - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "stream_descriptor": { - "name": "blogs" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "updated_at": "2027-07-11T13:07:45-07:00", - "blogs": { - "id": 99999999999999 - } - }, - "stream_descriptor": { - "name": "metafield_blogs" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "customers": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_customers" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "orders": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_orders" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "draft_orders": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_draft_orders" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "products": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_products" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "id": 99999999999999, - "products": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - } - }, - "stream_descriptor": { - "name": "product_images" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "products": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_product_images" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "id": 99999999999999, - "products": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - } - }, - "stream_descriptor": { - "name": "product_variants" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "products": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_product_variants" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "collects": { - "id": 99999999999999 - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "collections" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "collects": { - "id": 99999999999999 - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_collections" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "smart_collections" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "smart_collections": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_smart_collections" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "pages": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_pages" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_shops" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "locations": {}, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_locations" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": {}, - "stream_descriptor": { - "name": "disputes" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": {}, - "stream_descriptor": { - "name": "customer_saved_search" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "id": 99999999999999, - "customers": { - "updated_at": "2027-07-11T13:07:45-07:00" - } - }, - "stream_descriptor": { - "name": "customer_address" - } - } + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "customers" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "orders" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "draft_orders" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "products" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updatedAt": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "products_graph_ql" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "abandoned_checkouts" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafields" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 99999999999999 + }, + "stream_descriptor": { + "name": "collects" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "custom_collections" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "created_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "order_refunds" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "id": 99999999999999 + }, + "stream_descriptor": { + "name": "order_risks" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "created_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "transactions" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "processed_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "tender_transactions" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "pages" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "price_rules" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "price_rules": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "discount_codes" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "inventory_items" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "locations": {}, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "inventory_levels" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "id": 99999999999999 + }, + "stream_descriptor": { + "name": "fulfillment_orders" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "fulfillments" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": {}, + "stream_descriptor": { + "name": "balance_transactions" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 99999999999999, + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "stream_descriptor": { + "name": "articles" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "articles": { + "id": 99999999999999, + "deleted": {} + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_articles" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 99999999999999, + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "stream_descriptor": { + "name": "blogs" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2027-07-11T13:07:45-07:00", + "blogs": { + "id": 99999999999999 + } + }, + "stream_descriptor": { + "name": "metafield_blogs" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "customers": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_customers" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_orders" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "draft_orders": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_draft_orders" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_products" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 99999999999999, + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + } + }, + "stream_descriptor": { + "name": "product_images" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_product_images" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 99999999999999, + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + } + }, + "stream_descriptor": { + "name": "product_variants" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_product_variants" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "collects": { + "id": 99999999999999 + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "collections" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "collects": { + "id": 99999999999999 + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_collections" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "smart_collections" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "smart_collections": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_smart_collections" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "pages": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_pages" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_shops" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "locations": {}, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_locations" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": {}, + "stream_descriptor": { + "name": "disputes" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": {}, + "stream_descriptor": { + "name": "customer_saved_search" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 99999999999999, + "customers": { + "updated_at": "2027-07-11T13:07:45-07:00" + } + }, + "stream_descriptor": { + "name": "customer_address" + } } + } ] diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json index f96c4878b03fe..25734860ba983 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json @@ -1,257 +1,257 @@ { + "customers": { + "updated_at": "2023-07-11T13:07:45-07:00" + }, + "orders": { + "deleted": { + "deleted_at": "" + }, + "updated_at": "2023-04-24T11:00:10-07:00" + }, + "draft_orders": { + "updated_at": "2023-07-11T12:57:55-07:00" + }, + "products": { + "deleted": { + "deleted_at": "2023-09-05T13:32:22-07:00" + }, + "updated_at": "2023-09-05T07:12:05-07:00" + }, + "products_graph_ql": { + "updatedAt": "2023-09-05T14:12:05Z" + }, + "abandoned_checkouts": { + "updated_at": "2023-07-11T13:07:45-07:00" + }, + "metafields": { + "updated_at": "2022-05-30T23:42:02-07:00" + }, + "collects": { + "id": 29427031703741 + }, + "custom_collections": { + "deleted": { + "deleted_at": "2023-09-06T03:34:39-07:00" + }, + "updated_at": "2023-09-05T07:06:59-07:00" + }, + "order_refunds": { + "orders": { + "updated_at": "2023-04-24T11:00:10-07:00", + "deleted": { + "deleted_at": "" + } + }, + "created_at": "2022-10-10T06:21:53-07:00" + }, + "order_risks": { + "orders": { + "updated_at": "2023-04-24T07:03:06-07:00", + "deleted": { + "deleted_at": "" + } + }, + "id": 6446736474301 + }, + "transactions": { + "orders": { + "updated_at": "2023-04-24T11:00:10-07:00", + "deleted": { + "deleted_at": "" + } + }, + "created_at": "2023-04-24T11:00:08-07:00" + }, + "tender_transactions": { + "processed_at": "2023-04-24T11:00:08-07:00" + }, + "pages": { + "deleted": { + "deleted_at": "2023-09-06T03:37:06-07:00" + }, + "updated_at": "2023-04-24T11:08:41-07:00" + }, + "price_rules": { + "deleted": { + "deleted_at": "2023-09-06T03:48:46-07:00" + }, + "updated_at": "2023-04-24T05:52:22-07:00" + }, + "discount_codes": { + "price_rules": { + "updated_at": "2023-04-24T05:52:22-07:00", + "deleted": { + "deleted_at": "" + } + }, + "updated_at": "2023-04-24T05:52:22-07:00" + }, + "inventory_items": { + "products": { + "updated_at": "2023-04-20T04:12:51-07:00", + "deleted": { + "deleted_at": "" + } + }, + "updated_at": "2023-04-14T03:29:27-07:00" + }, + "inventory_levels": { + "locations": {}, + "updated_at": "2023-04-24T11:00:10-07:00" + }, + "fulfillment_orders": { + "orders": { + "updated_at": "2023-04-24T11:00:10-07:00", + "deleted": { + "deleted_at": "" + } + }, + "id": 5985636450493 + }, + "fulfillments": { + "orders": { + "updated_at": "2023-04-24T11:00:10-07:00", + "deleted": { + "deleted_at": "" + } + }, + "updated_at": "2023-04-24T11:00:09-07:00" + }, + "balance_transactions": {}, + "articles": { + "id": 558999404733, + "deleted": { + "deleted_at": "2023-09-05T14:02:00-07:00" + } + }, + "metafield_articles": { + "articles": { + "id": 558627979453, + "deleted": {} + }, + "updated_at": "2023-04-14T03:19:18-07:00" + }, + "blogs": { + "id": 85733114045, + "deleted": { + "deleted_at": "2023-09-06T03:30:22-07:00" + } + }, + "metafield_blogs": { + "updated_at": "2022-10-07T06:05:23-07:00", + "blogs": { + "id": 80417685693 + } + }, + "metafield_customers": { "customers": { - "updated_at": "2023-07-11T13:07:45-07:00" + "updated_at": "2023-04-24T06:53:48-07:00" }, + "updated_at": "2023-04-13T04:50:10-07:00" + }, + "metafield_orders": { "orders": { - "deleted": { - "deleted_at": "" - }, - "updated_at": "2023-04-24T11:00:10-07:00" - }, + "updated_at": "2023-04-24T10:59:00-07:00", + "deleted": { + "deleted_at": "" + } + }, + "updated_at": "2023-04-14T03:52:40-07:00" + }, + "metafield_draft_orders": { "draft_orders": { - "updated_at": "2023-07-11T12:57:55-07:00" + "updated_at": "2023-04-24T07:18:06-07:00" }, + "updated_at": "2023-04-24T07:18:06-07:00" + }, + "metafield_products": { "products": { - "deleted": { - "deleted_at": "2023-09-05T13:32:22-07:00" - }, - "updated_at": "2023-09-05T07:12:05-07:00" - }, - "products_graph_ql": { - "updatedAt": "2023-09-05T14:12:05Z" - }, - "abandoned_checkouts": { - "updated_at": "2023-07-11T13:07:45-07:00" - }, - "metafields": { - "updated_at": "2022-05-30T23:42:02-07:00" - }, + "updated_at": "2023-04-20T04:12:59-07:00", + "deleted": { + "deleted_at": "" + } + }, + "updated_at": "2023-04-14T04:04:46-07:00" + }, + "product_images": { + "id": 33290489659581, + "products": { + "updated_at": "", + "deleted": { + "deleted_at": "2023-09-05T13:32:22-07:00" + } + } + }, + "metafield_product_images": { + "products": { + "updated_at": "", + "deleted": { + "deleted_at": "2023-09-05T13:32:22-07:00" + } + }, + "updated_at": "2023-04-24T10:32:19-07:00" + }, + "product_variants": { + "id": 42778150305981, + "products": { + "updated_at": "", + "deleted": { + "deleted_at": "2023-09-05T13:32:22-07:00" + } + } + }, + "metafield_product_variants": { + "products": { + "updated_at": "", + "deleted": { + "deleted_at": "2023-09-05T13:32:22-07:00" + } + }, + "updated_at": "2023-04-14T03:29:27-07:00" + }, + "collections": { "collects": { - "id": 29427031703741 - }, - "custom_collections": { - "deleted": { - "deleted_at": "2023-09-06T03:34:39-07:00" - }, - "updated_at": "2023-09-05T07:06:59-07:00" - }, - "order_refunds": { - "orders": { - "updated_at": "2023-04-24T11:00:10-07:00", - "deleted": { - "deleted_at": "" - } - }, - "created_at": "2022-10-10T06:21:53-07:00" - }, - "order_risks": { - "orders": { - "updated_at": "2023-04-24T07:03:06-07:00", - "deleted": { - "deleted_at": "" - } - }, - "id": 6446736474301 - }, - "transactions": { - "orders": { - "updated_at": "2023-04-24T11:00:10-07:00", - "deleted": { - "deleted_at": "" - } - }, - "created_at": "2023-04-24T11:00:08-07:00" - }, - "tender_transactions": { - "processed_at": "2023-04-24T11:00:08-07:00" - }, - "pages": { - "deleted": { - "deleted_at": "2023-09-06T03:37:06-07:00" - }, - "updated_at": "2023-04-24T11:08:41-07:00" - }, - "price_rules": { - "deleted": { - "deleted_at": "2023-09-06T03:48:46-07:00" - }, - "updated_at": "2023-04-24T05:52:22-07:00" - }, - "discount_codes": { - "price_rules": { - "updated_at": "2023-04-24T05:52:22-07:00", - "deleted": { - "deleted_at": "" - } - }, - "updated_at": "2023-04-24T05:52:22-07:00" - }, - "inventory_items": { - "products": { - "updated_at": "2023-04-20T04:12:51-07:00", - "deleted": { - "deleted_at": "" - } - }, - "updated_at": "2023-04-14T03:29:27-07:00" - }, - "inventory_levels": { - "locations": {}, - "updated_at": "2023-04-24T11:00:10-07:00" - }, - "fulfillment_orders": { - "orders": { - "updated_at": "2023-04-24T11:00:10-07:00", - "deleted": { - "deleted_at": "" - } - }, - "id": 5985636450493 - }, - "fulfillments": { - "orders": { - "updated_at": "2023-04-24T11:00:10-07:00", - "deleted": { - "deleted_at": "" - } - }, - "updated_at": "2023-04-24T11:00:09-07:00" - }, - "balance_transactions": {}, - "articles": { - "id": 558999404733, - "deleted": { - "deleted_at": "2023-09-05T14:02:00-07:00" - } - }, - "metafield_articles": { - "articles": { - "id": 558627979453, - "deleted": {} - }, - "updated_at": "2023-04-14T03:19:18-07:00" - }, - "blogs": { - "id": 85733114045, - "deleted": { - "deleted_at": "2023-09-06T03:30:22-07:00" - } - }, - "metafield_blogs": { - "updated_at": "2022-10-07T06:05:23-07:00", - "blogs": { - "id": 80417685693 - } - }, - "metafield_customers": { - "customers": { - "updated_at": "2023-04-24T06:53:48-07:00" - }, - "updated_at": "2023-04-13T04:50:10-07:00" - }, - "metafield_orders": { - "orders": { - "updated_at": "2023-04-24T10:59:00-07:00", - "deleted": { - "deleted_at": "" - } - }, - "updated_at": "2023-04-14T03:52:40-07:00" - }, - "metafield_draft_orders": { - "draft_orders": { - "updated_at": "2023-04-24T07:18:06-07:00" - }, - "updated_at": "2023-04-24T07:18:06-07:00" - }, - "metafield_products": { - "products": { - "updated_at": "2023-04-20T04:12:59-07:00", - "deleted": { - "deleted_at": "" - } - }, - "updated_at": "2023-04-14T04:04:46-07:00" - }, - "product_images": { - "id": 33290489659581, - "products": { - "updated_at": "", - "deleted": { - "deleted_at": "2023-09-05T13:32:22-07:00" - } - } - }, - "metafield_product_images": { - "products": { - "updated_at": "", - "deleted": { - "deleted_at": "2023-09-05T13:32:22-07:00" - } - }, - "updated_at": "2023-04-24T10:32:19-07:00" - }, - "product_variants": { - "id": 42778150305981, - "products": { - "updated_at": "", - "deleted": { - "deleted_at": "2023-09-05T13:32:22-07:00" - } - } - }, - "metafield_product_variants": { - "products": { - "updated_at": "", - "deleted": { - "deleted_at": "2023-09-05T13:32:22-07:00" - } - }, - "updated_at": "2023-04-14T03:29:27-07:00" - }, - "collections": { - "collects": { - "id": 29427031703741 - }, - "updated_at": "2023-03-20T06:08:12-07:00" - }, - "metafield_collections": { - "collects": { - "id": 29427031703741 - }, - "updated_at": "2022-10-08T04:44:51-07:00" + "id": 29427031703741 }, + "updated_at": "2023-03-20T06:08:12-07:00" + }, + "metafield_collections": { + "collects": { + "id": 29427031703741 + }, + "updated_at": "2022-10-08T04:44:51-07:00" + }, + "smart_collections": { + "updated_at": "2023-09-05T07:12:04-07:00" + }, + "metafield_smart_collections": { "smart_collections": { - "updated_at": "2023-09-05T07:12:04-07:00" - }, - "metafield_smart_collections": { - "smart_collections": { - "updated_at": "2023-09-05T07:12:04-07:00" - }, - "updated_at": "2023-04-14T05:21:58-07:00" - }, - "metafield_pages": { - "pages": { - "updated_at": "2023-04-24T11:08:41-07:00", - "deleted": { - "deleted_at": "" - } - }, - "updated_at": "2023-04-24T11:08:41-07:00" - }, - "metafield_shops": { - "updated_at": "2023-04-24T11:12:38-07:00" + "updated_at": "2023-09-05T07:12:04-07:00" }, - "metafield_locations": { - "locations": {}, - "updated_at": "2022-10-12T02:21:35-07:00" - }, - "disputes": {}, - "customer_saved_search": {}, - "customer_address": { - "id": 8212915650749, - "customers": { - "updated_at": "2023-07-11T13:07:45-07:00" - } + "updated_at": "2023-04-14T05:21:58-07:00" + }, + "metafield_pages": { + "pages": { + "updated_at": "2023-04-24T11:08:41-07:00", + "deleted": { + "deleted_at": "" + } + }, + "updated_at": "2023-04-24T11:08:41-07:00" + }, + "metafield_shops": { + "updated_at": "2023-04-24T11:12:38-07:00" + }, + "metafield_locations": { + "locations": {}, + "updated_at": "2022-10-12T02:21:35-07:00" + }, + "disputes": {}, + "customer_saved_search": {}, + "customer_address": { + "id": 8212915650749, + "customers": { + "updated_at": "2023-07-11T13:07:45-07:00" } + } } diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json index 88686d74f32bc..8e298abbbbc6e 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json @@ -23,7 +23,7 @@ "float_ts": { "type": ["null", "number"] }, - "subtype": { + "subtype": { "type": ["null", "string"] }, "is_locked": { diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rate.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rate.json index 45b2339775bb7..5468844c8226e 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rate.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rate.json @@ -1,98 +1,50 @@ { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "object": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "active": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "country": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "created": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "display_name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "effective_percentage": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "inclusive": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "jurisdiction": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "livemode": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "metadata": { - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "percentage": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "state": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "tax_type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-trello/source_trello/manifest.yaml b/airbyte-integrations/connectors/source-trello/source_trello/manifest.yaml index 7f762698cfc8e..51b80752d8b51 100644 --- a/airbyte-integrations/connectors/source-trello/source_trello/manifest.yaml +++ b/airbyte-integrations/connectors/source-trello/source_trello/manifest.yaml @@ -49,12 +49,11 @@ definitions: pagination_strategy: type: CursorPagination page_size: 500 - cursor_value: '{{ (last_records|last)[''id''] }}' - stop_condition: '{{ not last_records }}' + cursor_value: "{{ (last_records|last)['id'] }}" + stop_condition: "{{ not last_records }}" board_id_partition_router: - type: CustomPartitionRouter - class_name: - source_trello.components.OrderIdsPartitionRouter + class_name: source_trello.components.OrderIdsPartitionRouter parent_stream_configs: - type: ParentStreamConfig parent_key: id @@ -76,7 +75,7 @@ definitions: $ref: "#/definitions/requester" path: members/me/boards request_parameters: - since: '{{ config[''start_date''] }}' + since: "{{ config['start_date'] }}" record_selector: $ref: "#/definitions/boards_selector" organizations_stream: @@ -89,7 +88,7 @@ definitions: $ref: "#/definitions/requester" path: members/me/organizations request_parameters: - since: '{{ config[''start_date''] }}' + since: "{{ config['start_date'] }}" record_selector: $ref: "#/definitions/selector" actions_stream: @@ -111,20 +110,20 @@ definitions: type: DatetimeBasedCursor cursor_field: date cursor_datetime_formats: - - '%Y-%m-%dT%H:%M:%S.%fZ' - datetime_format: '%Y-%m-%dT%H:%M:%S.%fZ' + - "%Y-%m-%dT%H:%M:%S.%fZ" + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" start_datetime: type: MinMaxDatetime - datetime: '{{ config[''start_date''] }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption field_name: since inject_into: request_parameter end_datetime: type: MinMaxDatetime - datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" cards_stream: type: DeclarativeStream name: cards @@ -135,13 +134,13 @@ definitions: $ref: "#/definitions/requester" path: boards/{{ stream_partition.id }}/cards/all request_parameters: - list: 'true' - sort: '-id' - since: '{{ config[''start_date''] }}' - members: 'true' - pluginData: 'true' - actions_display: 'true' - customFieldItems: 'true' + list: "true" + sort: "-id" + since: "{{ config['start_date'] }}" + members: "true" + pluginData: "true" + actions_display: "true" + customFieldItems: "true" record_selector: $ref: "#/definitions/selector" paginator: @@ -158,7 +157,7 @@ definitions: $ref: "#/definitions/requester" path: boards/{{ stream_partition.id }}/checklists request_parameters: - since: '{{ config[''start_date''] }}' + since: "{{ config['start_date'] }}" fields: all checkItem_fields: all record_selector: @@ -175,7 +174,7 @@ definitions: $ref: "#/definitions/requester" path: boards/{{ stream_partition.id }}/lists request_parameters: - since: '{{ config[''start_date''] }}' + since: "{{ config['start_date'] }}" record_selector: $ref: "#/definitions/selector" partition_router: @@ -190,7 +189,7 @@ definitions: $ref: "#/definitions/requester" path: boards/{{ stream_partition.id }}/members request_parameters: - since: '{{ config[''start_date''] }}' + since: "{{ config['start_date'] }}" record_selector: $ref: "#/definitions/selector" partition_router: @@ -235,7 +234,7 @@ spec: pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" description: UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. examples: - - '2021-03-01T00:00:00Z' + - "2021-03-01T00:00:00Z" format: date-time order: 2 board_ids: diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/deleted_tickets.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/deleted_tickets.json index e6e72140e9f7f..6f370225b0aa8 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/deleted_tickets.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/deleted_tickets.json @@ -6,13 +6,13 @@ "actor": { "type": ["null", "object"], "properties": { - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - } + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] } + } }, "id": { "type": ["null", "integer"] @@ -23,7 +23,7 @@ "description": { "type": ["null", "string"] }, - "deleted_at": { + "deleted_at": { "type": ["null", "string"], "format": "date-time" }, From db7fa52db8c667663886671028d526f831aec15f Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 22 Sep 2023 17:20:48 +0200 Subject: [PATCH 11/12] Add migration for config --- .../connectors/source-amazon-ads/main.py | 3 +- .../source_amazon_ads/config_migrations.py | 104 ++++++++++++++++++ .../unit_tests/test_source.py | 20 ++-- .../source-amazon-ads/unit_tests/utils.py | 2 + 4 files changed, 118 insertions(+), 11 deletions(-) create mode 100644 airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py diff --git a/airbyte-integrations/connectors/source-amazon-ads/main.py b/airbyte-integrations/connectors/source-amazon-ads/main.py index fa309cc66e047..69c638adc3a2d 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/main.py +++ b/airbyte-integrations/connectors/source-amazon-ads/main.py @@ -2,12 +2,13 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - import sys from airbyte_cdk.entrypoint import launch from source_amazon_ads import SourceAmazonAds +from source_amazon_ads.config_migrations import MigrateStartDate if __name__ == "__main__": source = SourceAmazonAds() + MigrateStartDate.migrate(sys.argv[1:], source) launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py new file mode 100644 index 0000000000000..dda920775164d --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py @@ -0,0 +1,104 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging +from typing import Any, List, Mapping + +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + +logger = logging.getLogger("airbyte_logger") + + +class MigrateStartDate: + """ + This class stands for migrating the config at runtime, + while providing the backward compatibility when falling back to the previous source version. + + Delete start_date field if it set to None or an empty string (""). + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + # Key used to identify start date in the configuration. + key: str = "start_date" + + @classmethod + def should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + Determines if a configuration requires migration. + + Args: + - config (Mapping[str, Any]): The configuration data to check. + + Returns: + - True: If the configuration requires migration. + - False: Otherwise. + """ + return not config.get(cls.key, "skip_if_start_date_in_config") + + @classmethod + def delete_from_config(cls, config: Mapping[str, Any], source: Source = None) -> Mapping[str, Any]: + """ + Removes the specified key from the configuration. + + Args: + - config (Mapping[str, Any]): The configuration from which the key should be removed. + - source (Source, optional): The data source. Defaults to None. + + Returns: + - Mapping[str, Any]: The configuration after removing the key. + """ + config.pop(cls.key, None) # Safely remove the key if it exists. + return config + + @classmethod + def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Modifies the configuration and then saves it back to the source. + + Args: + - config_path (str): The path where the configuration is stored. + - source (Source): The data source. + - config (Mapping[str, Any]): The current configuration. + + Returns: + - Mapping[str, Any]: The updated configuration. + """ + migrated_config = cls.delete_from_config(config, source) + source.write_config(migrated_config, config_path) + return migrated_config + + @classmethod + def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + """ + Emits the control messages related to configuration migration. + + Args: + - migrated_config (Mapping[str, Any]): The migrated configuration. + """ + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + for message in cls.message_repository._message_queue: + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: Source) -> None: + """ + Orchestrates the configuration migration process. + + It first checks if the `--config` argument is provided, and if so, + determines whether migration is needed, and then performs the migration + if required. + + Args: + - args (List[str]): List of command-line arguments. + - source (Source): The data source. + """ + config_path = AirbyteEntrypoint(source).extract_config(args) + if config_path: + config = source.read_config(config_path) + if cls.should_migrate(config): + cls.emit_control_message(cls.modify_and_save(config_path, source, config)) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py index d1e1a4f265c9f..5f5076edfbf78 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py @@ -2,8 +2,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import pytest import responses from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConnectorSpecification, Status, Type from jsonschema import Draft4Validator @@ -65,23 +63,25 @@ def test_check(config_gen): assert command_check(source, config_gen(start_date=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED) assert len(responses.calls) == 2 - with pytest.raises(Exception): - command_check(source, config_gen(start_date="")) + assert command_check(source, config_gen(start_date="")) == AirbyteConnectionStatus(status=Status.SUCCEEDED) + assert len(responses.calls) == 4 assert source.check(None, config_gen(start_date="2022-02-20")) == AirbyteConnectionStatus(status=Status.SUCCEEDED) - assert len(responses.calls) == 4 + assert len(responses.calls) == 6 assert command_check(source, config_gen(start_date="2022-20-02")) == AirbyteConnectionStatus( status=Status.FAILED, message="'month must be in 1..12'" ) - assert len(responses.calls) == 4 + assert len(responses.calls) == 6 - with pytest.raises(Exception): - command_check(source, config_gen(start_date="no date")) + assert command_check(source, config_gen(start_date="no date")) == AirbyteConnectionStatus( + status=Status.FAILED, message="'String does not match format YYYY-MM-DD'" + ) + assert len(responses.calls) == 6 assert command_check(source, config_gen(region=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED) - assert len(responses.calls) == 6 - assert url_strip_query(responses.calls[5].request.url) == "https://advertising-api.amazon.com/v2/profiles" + assert len(responses.calls) == 8 + assert url_strip_query(responses.calls[7].request.url) == "https://advertising-api.amazon.com/v2/profiles" assert command_check(source, config_gen(look_back_window=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py index 2b69cd22e4e4c..eb6cbb4dd93fd 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config +from source_amazon_ads.config_migrations import MigrateStartDate def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, Any]) -> Iterator[dict]: @@ -41,6 +42,7 @@ def read_full_refresh(stream_instance: Stream): def command_check(source: Source, config): logger = mock.MagicMock() connector_config, _ = split_config(config) + connector_config = MigrateStartDate.modify_and_save("unit_tests/config.json", source, connector_config) if source.check_config_against_spec: source_spec: ConnectorSpecification = source.spec(logger) check_config_against_spec_or_exit(connector_config, source_spec) From 535d1b74be4edb4f03c6028832f61a49e7722d2b Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 22 Sep 2023 17:23:44 +0200 Subject: [PATCH 12/12] Fix typo --- airbyte-integrations/connectors/source-amazon-ads/main.py | 1 + .../connectors/source-amazon-ads/unit_tests/test_source.py | 1 + 2 files changed, 2 insertions(+) diff --git a/airbyte-integrations/connectors/source-amazon-ads/main.py b/airbyte-integrations/connectors/source-amazon-ads/main.py index 69c638adc3a2d..1c292d29e4cad 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/main.py +++ b/airbyte-integrations/connectors/source-amazon-ads/main.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import sys from airbyte_cdk.entrypoint import launch diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py index 5f5076edfbf78..3dc310d11b250 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import responses from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConnectorSpecification, Status, Type from jsonschema import Draft4Validator