From 7bfc8b30c6f788ccc8110648708d33c1a2126397 Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Tue, 26 Sep 2023 14:42:31 -0700 Subject: [PATCH 1/8] Bump block size to 30 MB --- .../integrations/destination_async/GlobalMemoryManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/GlobalMemoryManager.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/GlobalMemoryManager.java index 90d6284d76c00..c79b78f9335d7 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/GlobalMemoryManager.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/GlobalMemoryManager.java @@ -38,7 +38,7 @@ public class GlobalMemoryManager { // In cases where a queue is rapidly expanding, a larger block size allows less allocation calls. On // the flip size, a smaller block size allows more granular memory management. Since this overhead // is minimal for now, err on a smaller block sizes. - public static final long BLOCK_SIZE_BYTES = 10 * 1024 * 1024; // 10MB + public static final long BLOCK_SIZE_BYTES = 30 * 1024 * 1024; // 30MB private final long maxMemoryBytes; private final AtomicLong currentMemoryBytes = new AtomicLong(0); @@ -61,7 +61,7 @@ public long getCurrentMemoryBytes() { * @return the size of the allocated block, in bytes */ public synchronized long requestMemory() { - // todo(davin): what happens if the incoming record is larger than 10MB? + // todo(davin): what happens if the incoming record is larger than 30MB? if (currentMemoryBytes.get() >= maxMemoryBytes) { return 0L; } From 3ee18525175fc30e8caadf49d1a2ebbf1690de93 Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Tue, 26 Sep 2023 14:45:57 -0700 Subject: [PATCH 2/8] Bump jackson version --- deps.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.toml b/deps.toml index d6187e55b7acd..a7e67a6d8021d 100644 --- a/deps.toml +++ b/deps.toml @@ -3,7 +3,7 @@ airbyte-protocol = "0.3.6" commons_io = "2.7" testcontainers = "1.19.0" datadog-version = "0.111.0" -fasterxml_version = "2.14.0" +fasterxml_version = "2.15.2" flyway = "7.14.0" glassfish_version = "2.31" hikaricp = "5.0.1" From 599626c5fdee49d49a0cf146de98f30824bfa3bd Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Tue, 26 Sep 2023 15:12:04 -0700 Subject: [PATCH 3/8] Increase block size --- .../destination-bigquery/Dockerfile | 2 +- .../destination-bigquery/metadata.yaml | 2 +- .../destination-redshift/Dockerfile | 2 +- .../destination-redshift/metadata.yaml | 2 +- .../destination-snowflake/Dockerfile | 2 +- .../destination-snowflake/metadata.yaml | 2 +- docs/integrations/destinations/bigquery.md | 231 +++++++++--------- docs/integrations/destinations/redshift.md | 1 + docs/integrations/destinations/snowflake.md | 1 + 9 files changed, 124 insertions(+), 121 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index 4a1158e3adfef..ef2cdee8024f2 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -25,5 +25,5 @@ ENV AIRBYTE_NORMALIZATION_INTEGRATION bigquery COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.0.16 +LABEL io.airbyte.version=2.0.17 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml index c3aaa7800d8f6..6d46e76fdabe4 100644 --- a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml +++ b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 - dockerImageTag: 2.0.16 + dockerImageTag: 2.0.17 dockerRepository: airbyte/destination-bigquery githubIssueLabel: destination-bigquery icon: bigquery.svg diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index 85e41effc4381..43ddf76072c2a 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -46,7 +46,7 @@ ENV APPLICATION destination-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.5 +LABEL io.airbyte.version=0.6.6 LABEL io.airbyte.name=airbyte/destination-redshift ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" diff --git a/airbyte-integrations/connectors/destination-redshift/metadata.yaml b/airbyte-integrations/connectors/destination-redshift/metadata.yaml index 0b7d33a37cedb..70db611d5f6dc 100644 --- a/airbyte-integrations/connectors/destination-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/destination-redshift/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc - dockerImageTag: 0.6.5 + dockerImageTag: 0.6.6 dockerRepository: airbyte/destination-redshift githubIssueLabel: destination-redshift icon: redshift.svg diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index 53b4cb64158fd..4349d5934082a 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -29,5 +29,5 @@ RUN tar xf ${APPLICATION}.tar --strip-components=1 ENV ENABLE_SENTRY true -LABEL io.airbyte.version=3.1.12 +LABEL io.airbyte.version=3.1.13 LABEL io.airbyte.name=airbyte/destination-snowflake diff --git a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml index d577400312a71..08819c66ab02c 100644 --- a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml +++ b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 424892c4-daac-4491-b35d-c6688ba547ba - dockerImageTag: 3.1.12 + dockerImageTag: 3.1.13 dockerRepository: airbyte/destination-snowflake githubIssueLabel: destination-snowflake icon: snowflake.svg diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index bc8973dae0983..0b2fc814ddbe7 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -131,119 +131,120 @@ Now that you have set up the BigQuery destination connector, check out the follo ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 2.0.16 | 2023-09-22 | [\#30697](https://github.com/airbytehq/airbyte/pull/30697) | Improve resiliency to unclean exit during schema change | -| 2.0.15 | 2023-09-21 | [\#30640](https://github.com/airbytehq/airbyte/pull/30640) | Handle streams with identical name and namespace | -| 2.0.14 | 2023-09-20 | [\#30069](https://github.com/airbytehq/airbyte/pull/30069) | Staging destination async | -| 2.0.13 | 2023-09-19 | [\#30592](https://github.com/airbytehq/airbyte/pull/30592) | Internal code changes | -| 2.0.12 | 2023-09-19 | [\#30319](https://github.com/airbytehq/airbyte/pull/30319) | Improved testing | -| 2.0.11 | 2023-09-18 | [\#30551](https://github.com/airbytehq/airbyte/pull/30551) | GCS Staging is first loading method option | -| 2.0.10 | 2023-09-15 | [\#30491](https://github.com/airbytehq/airbyte/pull/30491) | Improve error message display | -| 2.0.9 | 2023-09-14 | [\#30439](https://github.com/airbytehq/airbyte/pull/30439) | Fix a transient error | -| 2.0.8 | 2023-09-12 | [\#30364](https://github.com/airbytehq/airbyte/pull/30364) | Add log message | -| 2.0.7 | 2023-08-29 | [\#29878](https://github.com/airbytehq/airbyte/pull/29878) | Internal code changes | -| 2.0.6 | 2023-09-05 | [\#29917](https://github.com/airbytehq/airbyte/pull/29917) | Improve performance by changing metadata error array construction from ARRAY_CONCAT to ARRAY_AGG | -| 2.0.5 | 2023-08-31 | [\#30020](https://github.com/airbytehq/airbyte/pull/30020) | Run typing and deduping tasks in parallel | -| 2.0.4 | 2023-09-05 | [\#30117](https://github.com/airbytehq/airbyte/pull/30117) | Type and Dedupe at sync start and then every 6 hours | -| 2.0.3 | 2023-09-01 | [\#30056](https://github.com/airbytehq/airbyte/pull/30056) | Internal refactor, no behavior change | -| 2.0.2 | 2023-09-01 | [\#30120](https://github.com/airbytehq/airbyte/pull/30120) | Improve performance on very wide streams by skipping SAFE_CAST on strings | -| 2.0.1 | 2023-08-29 | [\#29972](https://github.com/airbytehq/airbyte/pull/29972) | Publish a new version to supersede old v2.0.0 | -| 2.0.0 | 2023-08-27 | [\#29783](https://github.com/airbytehq/airbyte/pull/29783) | Destinations V2 | -| 1.10.2 | 2023-08-24 | [\#29805](https://github.com/airbytehq/airbyte/pull/29805) | Destinations v2: Don't soft reset in migration | -| 1.10.1 | 2023-08-23 | [\#29774](https://github.com/airbytehq/airbyte/pull/29774) | Destinations v2: Don't soft reset overwrite syncs | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.0.17 | 2023-09-26 | [\#30775](https://github.com/airbytehq/airbyte/pull/30775 | Increase async block size +| 2.0.16 | 2023-09-22 | [\#30697](https://github.com/airbytehq/airbyte/pull/30697) | Improve resiliency to unclean exit during schema change | +| 2.0.15 | 2023-09-21 | [\#30640](https://github.com/airbytehq/airbyte/pull/30640) | Handle streams with identical name and namespace | +| 2.0.14 | 2023-09-20 | [\#30069](https://github.com/airbytehq/airbyte/pull/30069) | Staging destination async | +| 2.0.13 | 2023-09-19 | [\#30592](https://github.com/airbytehq/airbyte/pull/30592) | Internal code changes | +| 2.0.12 | 2023-09-19 | [\#30319](https://github.com/airbytehq/airbyte/pull/30319) | Improved testing | +| 2.0.11 | 2023-09-18 | [\#30551](https://github.com/airbytehq/airbyte/pull/30551) | GCS Staging is first loading method option | +| 2.0.10 | 2023-09-15 | [\#30491](https://github.com/airbytehq/airbyte/pull/30491) | Improve error message display | +| 2.0.9 | 2023-09-14 | [\#30439](https://github.com/airbytehq/airbyte/pull/30439) | Fix a transient error | +| 2.0.8 | 2023-09-12 | [\#30364](https://github.com/airbytehq/airbyte/pull/30364) | Add log message | +| 2.0.7 | 2023-08-29 | [\#29878](https://github.com/airbytehq/airbyte/pull/29878) | Internal code changes | +| 2.0.6 | 2023-09-05 | [\#29917](https://github.com/airbytehq/airbyte/pull/29917) | Improve performance by changing metadata error array construction from ARRAY_CONCAT to ARRAY_AGG | +| 2.0.5 | 2023-08-31 | [\#30020](https://github.com/airbytehq/airbyte/pull/30020) | Run typing and deduping tasks in parallel | +| 2.0.4 | 2023-09-05 | [\#30117](https://github.com/airbytehq/airbyte/pull/30117) | Type and Dedupe at sync start and then every 6 hours | +| 2.0.3 | 2023-09-01 | [\#30056](https://github.com/airbytehq/airbyte/pull/30056) | Internal refactor, no behavior change | +| 2.0.2 | 2023-09-01 | [\#30120](https://github.com/airbytehq/airbyte/pull/30120) | Improve performance on very wide streams by skipping SAFE_CAST on strings | +| 2.0.1 | 2023-08-29 | [\#29972](https://github.com/airbytehq/airbyte/pull/29972) | Publish a new version to supersede old v2.0.0 | +| 2.0.0 | 2023-08-27 | [\#29783](https://github.com/airbytehq/airbyte/pull/29783) | Destinations V2 | +| 1.10.2 | 2023-08-24 | [\#29805](https://github.com/airbytehq/airbyte/pull/29805) | Destinations v2: Don't soft reset in migration | +| 1.10.1 | 2023-08-23 | [\#29774](https://github.com/airbytehq/airbyte/pull/29774) | Destinations v2: Don't soft reset overwrite syncs | | 1.10.0 | 2023-08-21 | [\#29636](https://github.com/airbytehq/airbyte/pull/29636) | Destinations v2: Several Critical Bug Fixes (cursorless dedup, improved floating-point handling, improved special characters handling; improved error handling) | -| 1.9.1 | 2023-08-21 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Under the hood: Add dependency on Java CDK v0.0.1. | -| 1.9.0 | 2023-08-17 | [\#29560](https://github.com/airbytehq/airbyte/pull/29560) | Destinations v2: throw an error on disallowed column name prefixes | -| 1.8.1 | 2023-08-17 | [\#29522](https://github.com/airbytehq/airbyte/pull/29522) | Migration BugFix - ensure raw dataset created | -| 1.8.0 | 2023-08-17 | [\#29498](https://github.com/airbytehq/airbyte/pull/29498) | Fix checkpointing logic in GCS staging mode | -| 1.7.8 | 2023-08-15 | [\#29461](https://github.com/airbytehq/airbyte/pull/29461) | Migration BugFix - ensure migration happens before table creation for GCS staging. | -| 1.7.7 | 2023-08-11 | [\#29381](https://github.com/airbytehq/airbyte/pull/29381) | Destinations v2: Add support for streams with no columns | -| 1.7.6 | 2023-08-04 | [\#28894](https://github.com/airbytehq/airbyte/pull/28894) | Destinations v2: Add v1 -> v2 migration Logic | -| 1.7.5 | 2023-08-04 | [\#29106](https://github.com/airbytehq/airbyte/pull/29106) | Destinations v2: handle unusual CDC deletion edge case | -| 1.7.4 | 2023-08-04 | [\#29089](https://github.com/airbytehq/airbyte/pull/29089) | Destinations v2: improve special character handling in column names | -| 1.7.3 | 2023-08-03 | [\#28890](https://github.com/airbytehq/airbyte/pull/28890) | Internal code updates; improved testing | -| 1.7.2 | 2023-08-02 | [\#28976](https://github.com/airbytehq/airbyte/pull/28976) | Fix composite PK handling in v1 mode | -| 1.7.1 | 2023-08-02 | [\#28959](https://github.com/airbytehq/airbyte/pull/28959) | Destinations v2: Fix CDC syncs in non-dedup mode | -| 1.7.0 | 2023-08-01 | [\#28894](https://github.com/airbytehq/airbyte/pull/28894) | Destinations v2: Open up early access program opt-in | -| 1.6.0 | 2023-07-26 | [\#28723](https://github.com/airbytehq/airbyte/pull/28723) | Destinations v2: Change raw table dataset and naming convention | -| 1.5.8 | 2023-07-25 | [\#28721](https://github.com/airbytehq/airbyte/pull/28721) | Destinations v2: Handle cursor change across syncs | -| 1.5.7 | 2023-07-24 | [\#28625](https://github.com/airbytehq/airbyte/pull/28625) | Destinations v2: Limit Clustering Columns to 4 | -| 1.5.6 | 2023-07-21 | [\#28580](https://github.com/airbytehq/airbyte/pull/28580) | Destinations v2: Create dataset in user-specified location | -| 1.5.5 | 2023-07-20 | [\#28490](https://github.com/airbytehq/airbyte/pull/28490) | Destinations v2: Fix schema change detection in OVERWRITE mode when existing table is empty; other code refactoring | -| 1.5.4 | 2023-07-17 | [\#28382](https://github.com/airbytehq/airbyte/pull/28382) | Destinations v2: Schema Change Detection | -| 1.5.3 | 2023-07-14 | [\#28345](https://github.com/airbytehq/airbyte/pull/28345) | Increment patch to trigger a rebuild | -| 1.5.2 | 2023-07-05 | [\#27936](https://github.com/airbytehq/airbyte/pull/27936) | Internal scaffolding change for future development | -| 1.5.1 | 2023-06-30 | [\#27891](https://github.com/airbytehq/airbyte/pull/27891) | Revert bugged update | -| 1.5.0 | 2023-06-27 | [\#27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2 | -| 1.4.6 | 2023-06-28 | [\#27268](https://github.com/airbytehq/airbyte/pull/27268) | Internal scaffolding change for future development | -| 1.4.5 | 2023-06-21 | [\#27555](https://github.com/airbytehq/airbyte/pull/27555) | Reduce image size | -| 1.4.4 | 2023-05-25 | [\#26585](https://github.com/airbytehq/airbyte/pull/26585) | Small tweak in logs for clarity | -| 1.4.3 | 2023-05-17 | [\#26213](https://github.com/airbytehq/airbyte/pull/26213) | Fix bug in parsing file buffer config count | -| 1.4.2 | 2023-05-10 | [\#25925](https://github.com/airbytehq/airbyte/pull/25925) | Testing update. Normalization tests are now done in the destination container. | -| 1.4.1 | 2023-05-11 | [\#25993](https://github.com/airbytehq/airbyte/pull/25993) | Internal library update | -| 1.4.0 | 2023-04-29 | [\#25570](https://github.com/airbytehq/airbyte/pull/25570) | Internal library update. Bumping version to stay in sync with BigQuery-denormalized. | -| 1.3.4 | 2023-04-28 | [\#25588](https://github.com/airbytehq/airbyte/pull/25588) | Internal scaffolding change for future development | -| 1.3.3 | 2023-04-27 | [\#25346](https://github.com/airbytehq/airbyte/pull/25346) | Internal code cleanup | -| 1.3.1 | 2023-04-20 | [\#25097](https://github.com/airbytehq/airbyte/pull/25097) | Internal scaffolding change for future development | -| 1.3.0 | 2023-04-19 | [\#25287](https://github.com/airbytehq/airbyte/pull/25287) | Add parameter to configure the number of file buffers when GCS is used as the loading method | -| 1.2.20 | 2023-04-12 | [\#25122](https://github.com/airbytehq/airbyte/pull/25122) | Add additional data centers | -| 1.2.19 | 2023-03-29 | [\#24671](https://github.com/airbytehq/airbyte/pull/24671) | Fail faster in certain error cases | -| 1.2.18 | 2023-03-23 | [\#24447](https://github.com/airbytehq/airbyte/pull/24447) | Set the Service Account Key JSON field to always_show: true so that it isn't collapsed into an optional fields section | -| 1.2.17 | 2023-03-17 | [\#23788](https://github.com/airbytehq/airbyte/pull/23788) | S3-Parquet: added handler to process null values in arrays | -| 1.2.16 | 2023-03-10 | [\#23931](https://github.com/airbytehq/airbyte/pull/23931) | Added support for periodic buffer flush | -| 1.2.15 | 2023-03-10 | [\#23466](https://github.com/airbytehq/airbyte/pull/23466) | Changed S3 Avro type from Int to Long | -| 1.2.14 | 2023-02-08 | [\#22497](https://github.com/airbytehq/airbyte/pull/22497) | Fixed table already exists error | -| 1.2.13 | 2023-01-26 | [\#20631](https://github.com/airbytehq/airbyte/pull/20631) | Added support for destination checkpointing with staging | -| 1.2.12 | 2023-01-18 | [\#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | -| 1.2.11 | 2023-01-18 | [\#21144](https://github.com/airbytehq/airbyte/pull/21144) | Added explicit error message if sync fails due to a config issue | -| 1.2.9 | 2022-12-14 | [\#20501](https://github.com/airbytehq/airbyte/pull/20501) | Report GCS staging failures that occur during connection check | -| 1.2.8 | 2022-11-22 | [\#19489](https://github.com/airbytehq/airbyte/pull/19489) | Added non-billable projects handle to check connection stage | -| 1.2.7 | 2022-11-11 | [\#19358](https://github.com/airbytehq/airbyte/pull/19358) | Fixed check method to capture mismatch dataset location | -| 1.2.6 | 2022-11-10 | [\#18554](https://github.com/airbytehq/airbyte/pull/18554) | Improve check connection method to handle more errors | -| 1.2.5 | 2022-10-19 | [\#18162](https://github.com/airbytehq/airbyte/pull/18162) | Improve error logs | -| 1.2.4 | 2022-09-26 | [\#16890](https://github.com/airbytehq/airbyte/pull/16890) | Add user-agent header | -| 1.2.3 | 2022-09-22 | [\#17054](https://github.com/airbytehq/airbyte/pull/17054) | Respect stream namespace | -| 1.2.1 | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | (bugged, do not use) Wrap logs in AirbyteLogMessage | -| 1.2.0 | 2022-09-09 | [\#14023](https://github.com/airbytehq/airbyte/pull/14023) | (bugged, do not use) Cover arrays only if they are nested | -| 1.1.16 | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) | -| 1.1.15 | 2022-08-22 | [\#15787](https://github.com/airbytehq/airbyte/pull/15787) | Throw exception if job failed | -| 1.1.14 | 2022-08-03 | [\#14784](https://github.com/airbytehq/airbyte/pull/14784) | Enabling Application Default Credentials | -| 1.1.13 | 2022-08-02 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | -| 1.1.12 | 2022-08-02 | [\#15180](https://github.com/airbytehq/airbyte/pull/15180) | Fix standard loading mode | -| 1.1.11 | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | -| 1.1.10 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | -| 1.1.9 | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager | -| 1.1.8 | 2022-06-07 | [\#13579](https://github.com/airbytehq/airbyte/pull/13579) | Always check GCS bucket for GCS loading method to catch invalid HMAC keys. | -| 1.1.7 | 2022-06-07 | [\#13424](https://github.com/airbytehq/airbyte/pull/13424) | Reordered fields for specification. | -| 1.1.6 | 2022-05-15 | [\#12768](https://github.com/airbytehq/airbyte/pull/12768) | Clarify that the service account key json field is required on cloud. | -| 1.1.5 | 2022-05-12 | [\#12805](https://github.com/airbytehq/airbyte/pull/12805) | Updated to latest base-java to emit AirbyteTraceMessage on error. | -| 1.1.4 | 2022-05-04 | [\#12578](https://github.com/airbytehq/airbyte/pull/12578) | In JSON to Avro conversion, log JSON field values that do not follow Avro schema for debugging. | -| 1.1.3 | 2022-05-02 | [\#12528](https://github.com/airbytehq/airbyte/pull/12528) | Update Dataset location field description | -| 1.1.2 | 2022-04-29 | [\#12477](https://github.com/airbytehq/airbyte/pull/12477) | Dataset location is a required field | -| 1.1.1 | 2022-04-15 | [\#12068](https://github.com/airbytehq/airbyte/pull/12068) | Fixed bug with GCS bucket conditional binding | -| 1.1.0 | 2022-04-06 | [\#11776](https://github.com/airbytehq/airbyte/pull/11776) | Use serialized buffering strategy to reduce memory consumption. | -| 1.0.2 | 2022-03-30 | [\#11620](https://github.com/airbytehq/airbyte/pull/11620) | Updated spec | -| 1.0.1 | 2022-03-24 | [\#11350](https://github.com/airbytehq/airbyte/pull/11350) | Improve check performance | -| 1.0.0 | 2022-03-18 | [\#11238](https://github.com/airbytehq/airbyte/pull/11238) | Updated spec and documentation | -| 0.6.12 | 2022-03-18 | [\#10793](https://github.com/airbytehq/airbyte/pull/10793) | Fix namespace with invalid characters | -| 0.6.11 | 2022-03-03 | [\#10755](https://github.com/airbytehq/airbyte/pull/10755) | Make sure to kill children threads and stop JVM | -| 0.6.8 | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.6.6 | 2022-02-01 | [\#9959](https://github.com/airbytehq/airbyte/pull/9959) | Fix null pointer exception from buffered stream consumer. | -| 0.6.6 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | -| 0.6.5 | 2022-01-18 | [\#9573](https://github.com/airbytehq/airbyte/pull/9573) | BigQuery Destination : update description for some input fields | -| 0.6.4 | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | Support dataset-id prefixed by project-id | -| 0.6.3 | 2022-01-12 | [\#9415](https://github.com/airbytehq/airbyte/pull/9415) | BigQuery Destination : Fix GCS processing of Facebook data | -| 0.6.2 | 2022-01-10 | [\#9121](https://github.com/airbytehq/airbyte/pull/9121) | Fixed check method for GCS mode to verify if all roles assigned to user | -| 0.6.1 | 2021-12-22 | [\#9039](https://github.com/airbytehq/airbyte/pull/9039) | Added part_size configuration to UI for GCS staging | -| 0.6.0 | 2021-12-17 | [\#8788](https://github.com/airbytehq/airbyte/issues/8788) | BigQuery/BiqQuery denorm Destinations : Add possibility to use different types of GCS files | -| 0.5.1 | 2021-12-16 | [\#8816](https://github.com/airbytehq/airbyte/issues/8816) | Update dataset locations | -| 0.5.0 | 2021-10-26 | [\#7240](https://github.com/airbytehq/airbyte/issues/7240) | Output partitioned/clustered tables | -| 0.4.1 | 2021-10-04 | [\#6733](https://github.com/airbytehq/airbyte/issues/6733) | Support dataset starting with numbers | -| 0.4.0 | 2021-08-26 | [\#5296](https://github.com/airbytehq/airbyte/issues/5296) | Added GCS Staging uploading option | -| 0.3.12 | 2021-08-03 | [\#3549](https://github.com/airbytehq/airbyte/issues/3549) | Add optional arg to make a possibility to change the BigQuery client's chunk\buffer size | -| 0.3.11 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | -| 0.3.10 | 2021-07-28 | [\#3549](https://github.com/airbytehq/airbyte/issues/3549) | Add extended logs and made JobId filled with region and projectId | -| 0.3.9 | 2021-07-28 | [\#5026](https://github.com/airbytehq/airbyte/pull/5026) | Add sanitized json fields in raw tables to handle quotes in column names | -| 0.3.6 | 2021-06-18 | [\#3947](https://github.com/airbytehq/airbyte/issues/3947) | Service account credentials are now optional. | -| 0.3.4 | 2021-06-07 | [\#3277](https://github.com/airbytehq/airbyte/issues/3277) | Add dataset location option | +| 1.9.1 | 2023-08-21 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Under the hood: Add dependency on Java CDK v0.0.1. | +| 1.9.0 | 2023-08-17 | [\#29560](https://github.com/airbytehq/airbyte/pull/29560) | Destinations v2: throw an error on disallowed column name prefixes | +| 1.8.1 | 2023-08-17 | [\#29522](https://github.com/airbytehq/airbyte/pull/29522) | Migration BugFix - ensure raw dataset created | +| 1.8.0 | 2023-08-17 | [\#29498](https://github.com/airbytehq/airbyte/pull/29498) | Fix checkpointing logic in GCS staging mode | +| 1.7.8 | 2023-08-15 | [\#29461](https://github.com/airbytehq/airbyte/pull/29461) | Migration BugFix - ensure migration happens before table creation for GCS staging. | +| 1.7.7 | 2023-08-11 | [\#29381](https://github.com/airbytehq/airbyte/pull/29381) | Destinations v2: Add support for streams with no columns | +| 1.7.6 | 2023-08-04 | [\#28894](https://github.com/airbytehq/airbyte/pull/28894) | Destinations v2: Add v1 -> v2 migration Logic | +| 1.7.5 | 2023-08-04 | [\#29106](https://github.com/airbytehq/airbyte/pull/29106) | Destinations v2: handle unusual CDC deletion edge case | +| 1.7.4 | 2023-08-04 | [\#29089](https://github.com/airbytehq/airbyte/pull/29089) | Destinations v2: improve special character handling in column names | +| 1.7.3 | 2023-08-03 | [\#28890](https://github.com/airbytehq/airbyte/pull/28890) | Internal code updates; improved testing | +| 1.7.2 | 2023-08-02 | [\#28976](https://github.com/airbytehq/airbyte/pull/28976) | Fix composite PK handling in v1 mode | +| 1.7.1 | 2023-08-02 | [\#28959](https://github.com/airbytehq/airbyte/pull/28959) | Destinations v2: Fix CDC syncs in non-dedup mode | +| 1.7.0 | 2023-08-01 | [\#28894](https://github.com/airbytehq/airbyte/pull/28894) | Destinations v2: Open up early access program opt-in | +| 1.6.0 | 2023-07-26 | [\#28723](https://github.com/airbytehq/airbyte/pull/28723) | Destinations v2: Change raw table dataset and naming convention | +| 1.5.8 | 2023-07-25 | [\#28721](https://github.com/airbytehq/airbyte/pull/28721) | Destinations v2: Handle cursor change across syncs | +| 1.5.7 | 2023-07-24 | [\#28625](https://github.com/airbytehq/airbyte/pull/28625) | Destinations v2: Limit Clustering Columns to 4 | +| 1.5.6 | 2023-07-21 | [\#28580](https://github.com/airbytehq/airbyte/pull/28580) | Destinations v2: Create dataset in user-specified location | +| 1.5.5 | 2023-07-20 | [\#28490](https://github.com/airbytehq/airbyte/pull/28490) | Destinations v2: Fix schema change detection in OVERWRITE mode when existing table is empty; other code refactoring | +| 1.5.4 | 2023-07-17 | [\#28382](https://github.com/airbytehq/airbyte/pull/28382) | Destinations v2: Schema Change Detection | +| 1.5.3 | 2023-07-14 | [\#28345](https://github.com/airbytehq/airbyte/pull/28345) | Increment patch to trigger a rebuild | +| 1.5.2 | 2023-07-05 | [\#27936](https://github.com/airbytehq/airbyte/pull/27936) | Internal scaffolding change for future development | +| 1.5.1 | 2023-06-30 | [\#27891](https://github.com/airbytehq/airbyte/pull/27891) | Revert bugged update | +| 1.5.0 | 2023-06-27 | [\#27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2 | +| 1.4.6 | 2023-06-28 | [\#27268](https://github.com/airbytehq/airbyte/pull/27268) | Internal scaffolding change for future development | +| 1.4.5 | 2023-06-21 | [\#27555](https://github.com/airbytehq/airbyte/pull/27555) | Reduce image size | +| 1.4.4 | 2023-05-25 | [\#26585](https://github.com/airbytehq/airbyte/pull/26585) | Small tweak in logs for clarity | +| 1.4.3 | 2023-05-17 | [\#26213](https://github.com/airbytehq/airbyte/pull/26213) | Fix bug in parsing file buffer config count | +| 1.4.2 | 2023-05-10 | [\#25925](https://github.com/airbytehq/airbyte/pull/25925) | Testing update. Normalization tests are now done in the destination container. | +| 1.4.1 | 2023-05-11 | [\#25993](https://github.com/airbytehq/airbyte/pull/25993) | Internal library update | +| 1.4.0 | 2023-04-29 | [\#25570](https://github.com/airbytehq/airbyte/pull/25570) | Internal library update. Bumping version to stay in sync with BigQuery-denormalized. | +| 1.3.4 | 2023-04-28 | [\#25588](https://github.com/airbytehq/airbyte/pull/25588) | Internal scaffolding change for future development | +| 1.3.3 | 2023-04-27 | [\#25346](https://github.com/airbytehq/airbyte/pull/25346) | Internal code cleanup | +| 1.3.1 | 2023-04-20 | [\#25097](https://github.com/airbytehq/airbyte/pull/25097) | Internal scaffolding change for future development | +| 1.3.0 | 2023-04-19 | [\#25287](https://github.com/airbytehq/airbyte/pull/25287) | Add parameter to configure the number of file buffers when GCS is used as the loading method | +| 1.2.20 | 2023-04-12 | [\#25122](https://github.com/airbytehq/airbyte/pull/25122) | Add additional data centers | +| 1.2.19 | 2023-03-29 | [\#24671](https://github.com/airbytehq/airbyte/pull/24671) | Fail faster in certain error cases | +| 1.2.18 | 2023-03-23 | [\#24447](https://github.com/airbytehq/airbyte/pull/24447) | Set the Service Account Key JSON field to always_show: true so that it isn't collapsed into an optional fields section | +| 1.2.17 | 2023-03-17 | [\#23788](https://github.com/airbytehq/airbyte/pull/23788) | S3-Parquet: added handler to process null values in arrays | +| 1.2.16 | 2023-03-10 | [\#23931](https://github.com/airbytehq/airbyte/pull/23931) | Added support for periodic buffer flush | +| 1.2.15 | 2023-03-10 | [\#23466](https://github.com/airbytehq/airbyte/pull/23466) | Changed S3 Avro type from Int to Long | +| 1.2.14 | 2023-02-08 | [\#22497](https://github.com/airbytehq/airbyte/pull/22497) | Fixed table already exists error | +| 1.2.13 | 2023-01-26 | [\#20631](https://github.com/airbytehq/airbyte/pull/20631) | Added support for destination checkpointing with staging | +| 1.2.12 | 2023-01-18 | [\#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | +| 1.2.11 | 2023-01-18 | [\#21144](https://github.com/airbytehq/airbyte/pull/21144) | Added explicit error message if sync fails due to a config issue | +| 1.2.9 | 2022-12-14 | [\#20501](https://github.com/airbytehq/airbyte/pull/20501) | Report GCS staging failures that occur during connection check | +| 1.2.8 | 2022-11-22 | [\#19489](https://github.com/airbytehq/airbyte/pull/19489) | Added non-billable projects handle to check connection stage | +| 1.2.7 | 2022-11-11 | [\#19358](https://github.com/airbytehq/airbyte/pull/19358) | Fixed check method to capture mismatch dataset location | +| 1.2.6 | 2022-11-10 | [\#18554](https://github.com/airbytehq/airbyte/pull/18554) | Improve check connection method to handle more errors | +| 1.2.5 | 2022-10-19 | [\#18162](https://github.com/airbytehq/airbyte/pull/18162) | Improve error logs | +| 1.2.4 | 2022-09-26 | [\#16890](https://github.com/airbytehq/airbyte/pull/16890) | Add user-agent header | +| 1.2.3 | 2022-09-22 | [\#17054](https://github.com/airbytehq/airbyte/pull/17054) | Respect stream namespace | +| 1.2.1 | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | (bugged, do not use) Wrap logs in AirbyteLogMessage | +| 1.2.0 | 2022-09-09 | [\#14023](https://github.com/airbytehq/airbyte/pull/14023) | (bugged, do not use) Cover arrays only if they are nested | +| 1.1.16 | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) | +| 1.1.15 | 2022-08-22 | [\#15787](https://github.com/airbytehq/airbyte/pull/15787) | Throw exception if job failed | +| 1.1.14 | 2022-08-03 | [\#14784](https://github.com/airbytehq/airbyte/pull/14784) | Enabling Application Default Credentials | +| 1.1.13 | 2022-08-02 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | +| 1.1.12 | 2022-08-02 | [\#15180](https://github.com/airbytehq/airbyte/pull/15180) | Fix standard loading mode | +| 1.1.11 | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | +| 1.1.10 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | +| 1.1.9 | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager | +| 1.1.8 | 2022-06-07 | [\#13579](https://github.com/airbytehq/airbyte/pull/13579) | Always check GCS bucket for GCS loading method to catch invalid HMAC keys. | +| 1.1.7 | 2022-06-07 | [\#13424](https://github.com/airbytehq/airbyte/pull/13424) | Reordered fields for specification. | +| 1.1.6 | 2022-05-15 | [\#12768](https://github.com/airbytehq/airbyte/pull/12768) | Clarify that the service account key json field is required on cloud. | +| 1.1.5 | 2022-05-12 | [\#12805](https://github.com/airbytehq/airbyte/pull/12805) | Updated to latest base-java to emit AirbyteTraceMessage on error. | +| 1.1.4 | 2022-05-04 | [\#12578](https://github.com/airbytehq/airbyte/pull/12578) | In JSON to Avro conversion, log JSON field values that do not follow Avro schema for debugging. | +| 1.1.3 | 2022-05-02 | [\#12528](https://github.com/airbytehq/airbyte/pull/12528) | Update Dataset location field description | +| 1.1.2 | 2022-04-29 | [\#12477](https://github.com/airbytehq/airbyte/pull/12477) | Dataset location is a required field | +| 1.1.1 | 2022-04-15 | [\#12068](https://github.com/airbytehq/airbyte/pull/12068) | Fixed bug with GCS bucket conditional binding | +| 1.1.0 | 2022-04-06 | [\#11776](https://github.com/airbytehq/airbyte/pull/11776) | Use serialized buffering strategy to reduce memory consumption. | +| 1.0.2 | 2022-03-30 | [\#11620](https://github.com/airbytehq/airbyte/pull/11620) | Updated spec | +| 1.0.1 | 2022-03-24 | [\#11350](https://github.com/airbytehq/airbyte/pull/11350) | Improve check performance | +| 1.0.0 | 2022-03-18 | [\#11238](https://github.com/airbytehq/airbyte/pull/11238) | Updated spec and documentation | +| 0.6.12 | 2022-03-18 | [\#10793](https://github.com/airbytehq/airbyte/pull/10793) | Fix namespace with invalid characters | +| 0.6.11 | 2022-03-03 | [\#10755](https://github.com/airbytehq/airbyte/pull/10755) | Make sure to kill children threads and stop JVM | +| 0.6.8 | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.6.6 | 2022-02-01 | [\#9959](https://github.com/airbytehq/airbyte/pull/9959) | Fix null pointer exception from buffered stream consumer. | +| 0.6.6 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | +| 0.6.5 | 2022-01-18 | [\#9573](https://github.com/airbytehq/airbyte/pull/9573) | BigQuery Destination : update description for some input fields | +| 0.6.4 | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | Support dataset-id prefixed by project-id | +| 0.6.3 | 2022-01-12 | [\#9415](https://github.com/airbytehq/airbyte/pull/9415) | BigQuery Destination : Fix GCS processing of Facebook data | +| 0.6.2 | 2022-01-10 | [\#9121](https://github.com/airbytehq/airbyte/pull/9121) | Fixed check method for GCS mode to verify if all roles assigned to user | +| 0.6.1 | 2021-12-22 | [\#9039](https://github.com/airbytehq/airbyte/pull/9039) | Added part_size configuration to UI for GCS staging | +| 0.6.0 | 2021-12-17 | [\#8788](https://github.com/airbytehq/airbyte/issues/8788) | BigQuery/BiqQuery denorm Destinations : Add possibility to use different types of GCS files | +| 0.5.1 | 2021-12-16 | [\#8816](https://github.com/airbytehq/airbyte/issues/8816) | Update dataset locations | +| 0.5.0 | 2021-10-26 | [\#7240](https://github.com/airbytehq/airbyte/issues/7240) | Output partitioned/clustered tables | +| 0.4.1 | 2021-10-04 | [\#6733](https://github.com/airbytehq/airbyte/issues/6733) | Support dataset starting with numbers | +| 0.4.0 | 2021-08-26 | [\#5296](https://github.com/airbytehq/airbyte/issues/5296) | Added GCS Staging uploading option | +| 0.3.12 | 2021-08-03 | [\#3549](https://github.com/airbytehq/airbyte/issues/3549) | Add optional arg to make a possibility to change the BigQuery client's chunk\buffer size | +| 0.3.11 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | +| 0.3.10 | 2021-07-28 | [\#3549](https://github.com/airbytehq/airbyte/issues/3549) | Add extended logs and made JobId filled with region and projectId | +| 0.3.9 | 2021-07-28 | [\#5026](https://github.com/airbytehq/airbyte/pull/5026) | Add sanitized json fields in raw tables to handle quotes in column names | +| 0.3.6 | 2021-06-18 | [\#3947](https://github.com/airbytehq/airbyte/issues/3947) | Service account credentials are now optional. | +| 0.3.4 | 2021-06-07 | [\#3277](https://github.com/airbytehq/airbyte/issues/3277) | Add dataset location option | diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index 4dfedfd02f34f..e466891ad82f3 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -156,6 +156,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.6.6 | 2023-09-26 | [\#30775](https://github.com/airbytehq/airbyte/pull/30775 | Increase async block size | | 0.6.5 | 2023-08-18 | [\#28619](https://github.com/airbytehq/airbyte/pull/29640) | Fix duplicate staging object names in concurrent environment (e.g. async) | | 0.6.4 | 2023-08-10 | [\#28619](https://github.com/airbytehq/airbyte/pull/28619) | Use async method for staging | | 0.6.3 | 2023-08-07 | [\#29188](https://github.com/airbytehq/airbyte/pull/29188) | Internal code refactoring | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index ca63af67cdf00..24019eb2b4a70 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -271,6 +271,7 @@ Otherwise, make sure to grant the role the required permissions in the desired n | Version | Date | Pull Request | Subject | |:----------------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.1.13 | 2023-09-26 | [\#30775](https://github.com/airbytehq/airbyte/pull/30775 | Increase async block size | | 3.1.12 | 2023-09-21 | [\#30671](https://github.com/airbytehq/airbyte/pull/30671) | Reduce async buffer size | | 3.1.11 | 2023-09-19 | [\#30592](https://github.com/airbytehq/airbyte/pull/30592) | Internal code changes | | 3.1.10 | 2023-09-18 | [\#30546](https://github.com/airbytehq/airbyte/pull/30546) | Make sure that the async buffer are flush every 5 minutes | From 020c73fcee3d81f5086f5164aef3abef5305d5da Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Wed, 27 Sep 2023 09:14:46 -0700 Subject: [PATCH 4/8] build change --- .../bases/base-normalization/build.gradle | 159 +++++++++++------- 1 file changed, 94 insertions(+), 65 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index b73ddaabc2932..8768ac8dfe020 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -4,94 +4,125 @@ plugins { id 'airbyte-docker' id 'airbyte-python' } - airbytePython { moduleDirectory 'normalization' } - dependencies { - project(':airbyte-connector-test-harnesses:acceptance-test-harness') + implementation project(':airbyte-connector-test-harnesses:acceptance-test-harness') } - // we need to access the sshtunneling script from airbyte-workers for ssh support -def copySshScript = tasks.register('copySshScript', Copy) { +task copySshScript(type: Copy, dependsOn: [project(':airbyte-connector-test-harnesses:acceptance-test-harness').processResources]) { from "${project(':airbyte-connector-test-harnesses:acceptance-test-harness').buildDir}/resources/main" into "${buildDir}" include "sshtunneling.sh" } -copySshScript.configure { - dependsOn project(':airbyte-connector-test-harnesses:acceptance-test-harness').tasks.named('processResources') -} - // make sure the copy task above worked (if it fails, it fails silently annoyingly) -def checkSshScriptCopy = tasks.register('checkSshScriptCopy') { +task checkSshScriptCopy(type: Task, dependsOn: copySshScript) { doFirst { assert file("${buildDir}/sshtunneling.sh").exists() : "Copy of sshtunneling.sh failed, check that it is present in airbyte-workers." } } -checkSshScriptCopy.configure { - dependsOn copySshScript -} - -def generate = tasks.register('generate') -generate.configure { - dependsOn checkSshScriptCopy -} - -tasks.named('check').configure { - dependsOn generate -} - -def customIntegrationTestPython = tasks.register('customIntegrationTestPython', PythonTask) { - module = "pytest" - command = "-s integration_tests" -} -customIntegrationTestPython.configure { - dependsOn tasks.named('installTestReqs') - dependsOn tasks.named('assemble') -} - +airbyteDocker.dependsOn(checkSshScriptCopy) +assemble.dependsOn(checkSshScriptCopy) +test.dependsOn(checkSshScriptCopy) +integrationTest.dependsOn(build) static def getDockerfile(String customConnector) { return "${customConnector}.Dockerfile" } - static def getDockerImageName(String customConnector) { return "airbyte/normalization-${customConnector}" } - static def getImageNameWithTag(String customConnector) { return "${getDockerImageName(customConnector)}:dev" } -def customIntegrationTestsCoverage = tasks.named('_customIntegrationTestsCoverage') -customIntegrationTestsCoverage.configure { - dependsOn tasks.named('assemble') -} -[ - 'bigquery', - 'mysql', - 'postgres', - 'redshift', - 'snowflake', - 'oracle', - 'mssql', - 'clickhouse', - 'tidb', - 'duckdb', -].each {destinationName -> - def destinationProject = project(":airbyte-integrations:connectors:destination-$destinationName") - customIntegrationTestPython.configure { - dependsOn destinationProject.tasks.named('assemble') +def buildAirbyteDocker(String customConnector) { + // def baseCommand = ['docker', 'build', '.', '-f', getDockerfile(customConnector), '-t', getImageNameWithTag(customConnector)] + // As the base dbt image (https://hub.docker.com/r/fishtownanalytics/dbt/tags) we are using is only build for amd64, we need to use buildkit to force builds for your local environment + // We are lucky that all the python code dbt uses is mutli-arch compatible + + def arch = 'linux/amd64' + if (Os.isArch("aarch_64") || Os.isArch("aarch64")) { + arch = 'linux/arm64' } - // Not really sure what this task does differently from customIntegrationTestPython, - // but it seems to also run integration tests and as such it depends on the docker images. - customIntegrationTestsCoverage.configure { - dependsOn destinationProject.tasks.named('assemble') + + def baseCommand = ['docker', 'buildx', 'build', '--load', '--platform', arch, '-f', getDockerfile(customConnector), '-t', getImageNameWithTag(customConnector), '.'] + println("Building normalization container: " + baseCommand.join(" ")) + + return { + commandLine baseCommand } } - +task airbyteDockerMSSql(type: Exec, dependsOn: checkSshScriptCopy) { + configure buildAirbyteDocker('mssql') + dependsOn assemble +} +task airbyteDockerMySql(type: Exec, dependsOn: checkSshScriptCopy) { + configure buildAirbyteDocker('mysql') + dependsOn assemble +} +task airbyteDockerOracle(type: Exec, dependsOn: checkSshScriptCopy) { + configure buildAirbyteDocker('oracle') + dependsOn assemble +} +task airbyteDockerClickhouse(type: Exec, dependsOn: checkSshScriptCopy) { + configure buildAirbyteDocker('clickhouse') + dependsOn assemble +} +task airbyteDockerSnowflake(type: Exec, dependsOn: checkSshScriptCopy) { + configure buildAirbyteDocker('snowflake') + dependsOn assemble +} +task airbyteDockerRedshift(type: Exec, dependsOn: checkSshScriptCopy) { + configure buildAirbyteDocker('redshift') + dependsOn assemble +} +task airbyteDockerTiDB(type: Exec, dependsOn: checkSshScriptCopy) { + configure buildAirbyteDocker('tidb') + dependsOn assemble +} +task airbyteDockerDuckDB(type: Exec, dependsOn: checkSshScriptCopy) { + configure buildAirbyteDocker('duckdb') + dependsOn assemble +} +airbyteDocker.dependsOn(airbyteDockerMSSql) +airbyteDocker.dependsOn(airbyteDockerMySql) +airbyteDocker.dependsOn(airbyteDockerOracle) +airbyteDocker.dependsOn(airbyteDockerClickhouse) +airbyteDocker.dependsOn(airbyteDockerSnowflake) +airbyteDocker.dependsOn(airbyteDockerRedshift) +airbyteDocker.dependsOn(airbyteDockerTiDB) +airbyteDocker.dependsOn(airbyteDockerDuckDB) +task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs) { + module = "pytest" + command = "-s integration_tests" + dependsOn ':airbyte-integrations:bases:base-normalization:airbyteDocker' + dependsOn ':airbyte-integrations:connectors:destination-bigquery:airbyteDocker' + dependsOn ':airbyte-integrations:connectors:destination-mysql:airbyteDocker' + dependsOn ':airbyte-integrations:connectors:destination-postgres:airbyteDocker' + dependsOn ':airbyte-integrations:connectors:destination-redshift:airbyteDocker' + dependsOn ':airbyte-integrations:connectors:destination-snowflake:airbyteDocker' + dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker' + dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker' + dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker' + dependsOn ':airbyte-integrations:connectors:destination-tidb:airbyteDocker' + dependsOn ':airbyte-integrations:connectors:destination-duckdb:airbyteDocker' +} +// not really sure what this task does differently from customIntegrationTestPython, but it seems to also run integration tests +// and as such it depends on the docker images. +project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:bases:base-normalization:airbyteDocker' +project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-bigquery:airbyteDocker' +project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-mysql:airbyteDocker' +project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-postgres:airbyteDocker' +project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-redshift:airbyteDocker' +project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-snowflake:airbyteDocker' +project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker' +project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker' +project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker' +project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-tidb:airbyteDocker' +project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-duckdb:airbyteDocker' // DATs have some additional tests that exercise normalization code paths, // so we want to run these in addition to the base-normalization integration tests. // If you add more items here, make sure to also to have CI fetch their credentials. @@ -100,14 +131,12 @@ customIntegrationTestsCoverage.configure { // integrationTest.dependsOn(":airbyte-integrations:connectors:destination-bigquery:integrationTest") // integrationTest.dependsOn(":airbyte-integrations:connectors:destination-postgres:integrationTest") // integrationTest.dependsOn(":airbyte-integrations:connectors:destination-snowflake:integrationTest") - -tasks.named('customIntegrationTests').configure { - dependsOn customIntegrationTestPython -} - +integrationTest.dependsOn("customIntegrationTestPython") +customIntegrationTests.dependsOn("customIntegrationTestPython") // TODO fix and use https://github.com/airbytehq/airbyte/issues/3192 instead -def mypyCheck = tasks.register('mypyCheck', PythonTask) { +task('mypyCheck', type: PythonTask) { module = "mypy" command = "normalization --config-file ${project.rootProject.file('pyproject.toml').absolutePath}" + dependsOn 'blackFormat' } -tasks.named('check').configure { dependsOn mypyCheck } +check.dependsOn mypyCheck \ No newline at end of file From efac8cea370531fe21b5cefc04b0053e9ff8d818 Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Wed, 27 Sep 2023 10:15:45 -0700 Subject: [PATCH 5/8] Revert "build change" This reverts commit 020c73fcee3d81f5086f5164aef3abef5305d5da. --- .../bases/base-normalization/build.gradle | 159 +++++++----------- 1 file changed, 65 insertions(+), 94 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index 8768ac8dfe020..b73ddaabc2932 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -4,125 +4,94 @@ plugins { id 'airbyte-docker' id 'airbyte-python' } + airbytePython { moduleDirectory 'normalization' } + dependencies { - implementation project(':airbyte-connector-test-harnesses:acceptance-test-harness') + project(':airbyte-connector-test-harnesses:acceptance-test-harness') } + // we need to access the sshtunneling script from airbyte-workers for ssh support -task copySshScript(type: Copy, dependsOn: [project(':airbyte-connector-test-harnesses:acceptance-test-harness').processResources]) { +def copySshScript = tasks.register('copySshScript', Copy) { from "${project(':airbyte-connector-test-harnesses:acceptance-test-harness').buildDir}/resources/main" into "${buildDir}" include "sshtunneling.sh" } +copySshScript.configure { + dependsOn project(':airbyte-connector-test-harnesses:acceptance-test-harness').tasks.named('processResources') +} + // make sure the copy task above worked (if it fails, it fails silently annoyingly) -task checkSshScriptCopy(type: Task, dependsOn: copySshScript) { +def checkSshScriptCopy = tasks.register('checkSshScriptCopy') { doFirst { assert file("${buildDir}/sshtunneling.sh").exists() : "Copy of sshtunneling.sh failed, check that it is present in airbyte-workers." } } -airbyteDocker.dependsOn(checkSshScriptCopy) -assemble.dependsOn(checkSshScriptCopy) -test.dependsOn(checkSshScriptCopy) -integrationTest.dependsOn(build) +checkSshScriptCopy.configure { + dependsOn copySshScript +} + +def generate = tasks.register('generate') +generate.configure { + dependsOn checkSshScriptCopy +} + +tasks.named('check').configure { + dependsOn generate +} + +def customIntegrationTestPython = tasks.register('customIntegrationTestPython', PythonTask) { + module = "pytest" + command = "-s integration_tests" +} +customIntegrationTestPython.configure { + dependsOn tasks.named('installTestReqs') + dependsOn tasks.named('assemble') +} + static def getDockerfile(String customConnector) { return "${customConnector}.Dockerfile" } + static def getDockerImageName(String customConnector) { return "airbyte/normalization-${customConnector}" } + static def getImageNameWithTag(String customConnector) { return "${getDockerImageName(customConnector)}:dev" } +def customIntegrationTestsCoverage = tasks.named('_customIntegrationTestsCoverage') +customIntegrationTestsCoverage.configure { + dependsOn tasks.named('assemble') +} -def buildAirbyteDocker(String customConnector) { - // def baseCommand = ['docker', 'build', '.', '-f', getDockerfile(customConnector), '-t', getImageNameWithTag(customConnector)] - // As the base dbt image (https://hub.docker.com/r/fishtownanalytics/dbt/tags) we are using is only build for amd64, we need to use buildkit to force builds for your local environment - // We are lucky that all the python code dbt uses is mutli-arch compatible - - def arch = 'linux/amd64' - if (Os.isArch("aarch_64") || Os.isArch("aarch64")) { - arch = 'linux/arm64' +[ + 'bigquery', + 'mysql', + 'postgres', + 'redshift', + 'snowflake', + 'oracle', + 'mssql', + 'clickhouse', + 'tidb', + 'duckdb', +].each {destinationName -> + def destinationProject = project(":airbyte-integrations:connectors:destination-$destinationName") + customIntegrationTestPython.configure { + dependsOn destinationProject.tasks.named('assemble') } - - def baseCommand = ['docker', 'buildx', 'build', '--load', '--platform', arch, '-f', getDockerfile(customConnector), '-t', getImageNameWithTag(customConnector), '.'] - println("Building normalization container: " + baseCommand.join(" ")) - - return { - commandLine baseCommand + // Not really sure what this task does differently from customIntegrationTestPython, + // but it seems to also run integration tests and as such it depends on the docker images. + customIntegrationTestsCoverage.configure { + dependsOn destinationProject.tasks.named('assemble') } } -task airbyteDockerMSSql(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('mssql') - dependsOn assemble -} -task airbyteDockerMySql(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('mysql') - dependsOn assemble -} -task airbyteDockerOracle(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('oracle') - dependsOn assemble -} -task airbyteDockerClickhouse(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('clickhouse') - dependsOn assemble -} -task airbyteDockerSnowflake(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('snowflake') - dependsOn assemble -} -task airbyteDockerRedshift(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('redshift') - dependsOn assemble -} -task airbyteDockerTiDB(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('tidb') - dependsOn assemble -} -task airbyteDockerDuckDB(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('duckdb') - dependsOn assemble -} -airbyteDocker.dependsOn(airbyteDockerMSSql) -airbyteDocker.dependsOn(airbyteDockerMySql) -airbyteDocker.dependsOn(airbyteDockerOracle) -airbyteDocker.dependsOn(airbyteDockerClickhouse) -airbyteDocker.dependsOn(airbyteDockerSnowflake) -airbyteDocker.dependsOn(airbyteDockerRedshift) -airbyteDocker.dependsOn(airbyteDockerTiDB) -airbyteDocker.dependsOn(airbyteDockerDuckDB) -task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs) { - module = "pytest" - command = "-s integration_tests" - dependsOn ':airbyte-integrations:bases:base-normalization:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-bigquery:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-mysql:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-postgres:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-redshift:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-snowflake:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-tidb:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-duckdb:airbyteDocker' -} -// not really sure what this task does differently from customIntegrationTestPython, but it seems to also run integration tests -// and as such it depends on the docker images. -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:bases:base-normalization:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-bigquery:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-mysql:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-postgres:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-redshift:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-snowflake:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-tidb:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-duckdb:airbyteDocker' + // DATs have some additional tests that exercise normalization code paths, // so we want to run these in addition to the base-normalization integration tests. // If you add more items here, make sure to also to have CI fetch their credentials. @@ -131,12 +100,14 @@ project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte- // integrationTest.dependsOn(":airbyte-integrations:connectors:destination-bigquery:integrationTest") // integrationTest.dependsOn(":airbyte-integrations:connectors:destination-postgres:integrationTest") // integrationTest.dependsOn(":airbyte-integrations:connectors:destination-snowflake:integrationTest") -integrationTest.dependsOn("customIntegrationTestPython") -customIntegrationTests.dependsOn("customIntegrationTestPython") + +tasks.named('customIntegrationTests').configure { + dependsOn customIntegrationTestPython +} + // TODO fix and use https://github.com/airbytehq/airbyte/issues/3192 instead -task('mypyCheck', type: PythonTask) { +def mypyCheck = tasks.register('mypyCheck', PythonTask) { module = "mypy" command = "normalization --config-file ${project.rootProject.file('pyproject.toml').absolutePath}" - dependsOn 'blackFormat' } -check.dependsOn mypyCheck \ No newline at end of file +tasks.named('check').configure { dependsOn mypyCheck } From e4a6d581df40fe54790eb46775927a81cd29b559 Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Wed, 27 Sep 2023 22:18:42 +0000 Subject: [PATCH 6/8] Automated Commit - Formatting Changes --- .../source-greenhouse/source_greenhouse/schemas/candidates.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/candidates.json b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/candidates.json index 792531dc0d126..8e3f9d7a301ca 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/candidates.json +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/candidates.json @@ -219,7 +219,7 @@ "type": "array" }, "custom_fields": { - "properties": {}, + "properties": {}, "additionalProperties": true, "type": ["null", "object"] }, From 697defd003a02394a7f4666690a88426b0a8bed7 Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Wed, 27 Sep 2023 15:19:26 -0700 Subject: [PATCH 7/8] Remove redshift --- airbyte-integrations/connectors/destination-redshift/Dockerfile | 2 +- .../connectors/destination-redshift/metadata.yaml | 2 +- docs/integrations/destinations/redshift.md | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index 43ddf76072c2a..85e41effc4381 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -46,7 +46,7 @@ ENV APPLICATION destination-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.6 +LABEL io.airbyte.version=0.6.5 LABEL io.airbyte.name=airbyte/destination-redshift ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" diff --git a/airbyte-integrations/connectors/destination-redshift/metadata.yaml b/airbyte-integrations/connectors/destination-redshift/metadata.yaml index 70db611d5f6dc..0b7d33a37cedb 100644 --- a/airbyte-integrations/connectors/destination-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/destination-redshift/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc - dockerImageTag: 0.6.6 + dockerImageTag: 0.6.5 dockerRepository: airbyte/destination-redshift githubIssueLabel: destination-redshift icon: redshift.svg diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index e466891ad82f3..4dfedfd02f34f 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -156,7 +156,6 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.6.6 | 2023-09-26 | [\#30775](https://github.com/airbytehq/airbyte/pull/30775 | Increase async block size | | 0.6.5 | 2023-08-18 | [\#28619](https://github.com/airbytehq/airbyte/pull/29640) | Fix duplicate staging object names in concurrent environment (e.g. async) | | 0.6.4 | 2023-08-10 | [\#28619](https://github.com/airbytehq/airbyte/pull/28619) | Use async method for staging | | 0.6.3 | 2023-08-07 | [\#29188](https://github.com/airbytehq/airbyte/pull/29188) | Internal code refactoring | From 07102095d70f4f717cbc34d357dbb0a8b120847c Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Wed, 27 Sep 2023 22:56:51 +0000 Subject: [PATCH 8/8] Automated Commit - Formatting Changes --- .../destination/s3/S3DestinationStrictEncrypt.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncrypt.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncrypt.java index 0dbccd86f8757..8258bbcb7d04d 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncrypt.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncrypt.java @@ -6,9 +6,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; -import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfigFactory; -import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; import io.airbyte.cdk.integrations.destination.s3.S3BaseChecks; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfigFactory; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status;