From 5a835c2c6812bd4b0e18087d557150522e1a7b23 Mon Sep 17 00:00:00 2001 From: Ryan Fu Date: Wed, 5 Apr 2023 12:47:23 -0700 Subject: [PATCH] Revert "JDBC connections to support checkpointing (#24604)" This reverts commit 73900fad3ed6ce694643def55bf461f32351e77b. --- .../Dockerfile | 2 +- .../destination-clickhouse/Dockerfile | 2 +- .../jdbc/JdbcBufferedConsumerFactory.java | 108 +++++++++--------- .../staging/StagingConsumerFactory.java | 1 - .../Dockerfile | 2 +- .../connectors/destination-mssql/Dockerfile | 2 +- .../Dockerfile | 2 +- .../destination-postgres/Dockerfile | 2 +- .../connectors/destination-tidb/Dockerfile | 2 +- docs/integrations/destinations/clickhouse.md | 29 +++-- docs/integrations/destinations/mssql.md | 98 ++++++++-------- docs/integrations/destinations/postgres.md | 47 ++++---- docs/integrations/destinations/tidb.md | 7 +- 13 files changed, 149 insertions(+), 155 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile index c62d6a5792b901..8a6e418ae246b8 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.3 +LABEL io.airbyte.version=0.2.2 LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile index abd96df2014b79..604ed1f756d1e8 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.3 +LABEL io.airbyte.version=0.2.2 LABEL io.airbyte.name=airbyte/destination-clickhouse diff --git a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java index b6555f9d8db205..b7556444d148f8 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java @@ -35,15 +35,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/**Strategy: - *

1. Create a final table for each stream - *

2. Accumulate records in a buffer. One buffer per stream - *

3. As records accumulate write them in batch to the database. We set a minimum numbers of records - * before writing to avoid wasteful record-wise writes. In the case with slow syncs this will be - * superseded with a periodic record flush from {@link BufferedStreamConsumer#periodicBufferFlush()} - *

4. Once all records have been written to buffer, flush the buffer and write any remaining records - * to the database (regardless of how few are left) - */ +// Strategy: +// 1. Create a temporary table for each stream +// 2. Accumulate records in a buffer. One buffer per stream. +// 3. As records accumulate write them in batch to the database. We set a minimum numbers of records +// before writing to avoid wasteful record-wise writes. +// 4. Once all records have been written to buffer, flush the buffer and write any remaining records +// to the database (regardless of how few are left). +// 5. In a single transaction, delete the target tables if they exist and rename the temp tables to +// the final table name. public class JdbcBufferedConsumerFactory { private static final Logger LOGGER = LoggerFactory.getLogger(JdbcBufferedConsumerFactory.class); @@ -116,52 +116,24 @@ private static String getOutputSchema(final AirbyteStream stream, : namingResolver.getNamespace(defaultDestSchema); } - /** - * Sets up destination storage through: - *

1. Creates Schema (if not exists) - *

2. Creates airybte_raw table (if not exists) - *

3. Truncates table if sync mode is in OVERWRITE - * - * @param database JDBC database to connect to - * @param sqlOperations interface for execution SQL queries - * @param writeConfigs settings for each stream - * @return - */ private static OnStartFunction onStartFunction(final JdbcDatabase database, final SqlOperations sqlOperations, final List writeConfigs) { return () -> { - LOGGER.info("Preparing raw tables in destination started for {} streams", writeConfigs.size()); - final List queryList = new ArrayList<>(); + LOGGER.info("Preparing tmp tables in destination started for {} streams", writeConfigs.size()); for (final WriteConfig writeConfig : writeConfigs) { final String schemaName = writeConfig.getOutputSchemaName(); - final String dstTableName = writeConfig.getOutputTableName(); - LOGGER.info("Preparing raw table in destination started for stream {}. schema: {}, table name: {}", - writeConfig.getStreamName(), - schemaName, - dstTableName); + final String tmpTableName = writeConfig.getTmpTableName(); + LOGGER.info("Preparing tmp table in destination started for stream {}. schema: {}, tmp table name: {}", writeConfig.getStreamName(), + schemaName, tmpTableName); + sqlOperations.createSchemaIfNotExists(database, schemaName); - sqlOperations.createTableIfNotExists(database, schemaName, dstTableName); - switch (writeConfig.getSyncMode()) { - case OVERWRITE -> queryList.add(sqlOperations.truncateTableQuery(database, schemaName, dstTableName)); - case APPEND, APPEND_DEDUP -> {} - default -> throw new IllegalStateException("Unrecognized sync mode: " + writeConfig.getSyncMode()); - } + sqlOperations.createTableIfNotExists(database, schemaName, tmpTableName); } - sqlOperations.executeTransaction(database, queryList); - LOGGER.info("Preparing raw tables in destination completed."); + LOGGER.info("Preparing tables in destination completed."); }; } - /** - * Writes {@link AirbyteRecordMessage} to JDBC database's airbyte_raw table - * - * @param database JDBC database to connect to - * @param sqlOperations interface of SQL queries to execute - * @param writeConfigs settings for each stream - * @param catalog catalog of all streams to sync - * @return - */ private static RecordWriter recordWriterFunction(final JdbcDatabase database, final SqlOperations sqlOperations, final List writeConfigs, @@ -176,26 +148,54 @@ private static RecordWriter recordWriterFunction(final Jdb } final WriteConfig writeConfig = pairToWriteConfig.get(pair); - sqlOperations.insertRecords(database, records, writeConfig.getOutputSchemaName(), writeConfig.getOutputTableName()); + sqlOperations.insertRecords(database, records, writeConfig.getOutputSchemaName(), writeConfig.getTmpTableName()); }; } - /** - * Closes connection to JDBC database and other tear down functionality - * - * @param database JDBC database to connect to - * @param sqlOperations interface used to execute SQL queries - * @param writeConfigs settings for each stream - * @return - */ private static OnCloseFunction onCloseFunction(final JdbcDatabase database, final SqlOperations sqlOperations, final List writeConfigs) { return (hasFailed) -> { + // copy data if (!hasFailed) { + final List queryList = new ArrayList<>(); sqlOperations.onDestinationCloseOperations(database, writeConfigs); + LOGGER.info("Finalizing tables in destination started for {} streams", writeConfigs.size()); + for (final WriteConfig writeConfig : writeConfigs) { + final String schemaName = writeConfig.getOutputSchemaName(); + final String srcTableName = writeConfig.getTmpTableName(); + final String dstTableName = writeConfig.getOutputTableName(); + LOGGER.info("Finalizing stream {}. schema {}, tmp table {}, final table {}", writeConfig.getStreamName(), schemaName, srcTableName, + dstTableName); + + sqlOperations.createTableIfNotExists(database, schemaName, dstTableName); + switch (writeConfig.getSyncMode()) { + case OVERWRITE -> queryList.add(sqlOperations.truncateTableQuery(database, schemaName, dstTableName)); + case APPEND -> {} + case APPEND_DEDUP -> {} + default -> throw new IllegalStateException("Unrecognized sync mode: " + writeConfig.getSyncMode()); + } + queryList.add(sqlOperations.insertTableQuery(database, schemaName, srcTableName, dstTableName)); + } + + LOGGER.info("Executing finalization of tables."); + sqlOperations.executeTransaction(database, queryList); + LOGGER.info("Finalizing tables in destination completed."); } - }; + // clean up + LOGGER.info("Cleaning tmp tables in destination started for {} streams", writeConfigs.size()); + for (final WriteConfig writeConfig : writeConfigs) { + final String schemaName = writeConfig.getOutputSchemaName(); + final String tmpTableName = writeConfig.getTmpTableName(); + LOGGER.info("Cleaning tmp table in destination started for stream {}. schema {}, tmp table name: {}", writeConfig.getStreamName(), schemaName, + tmpTableName); + + sqlOperations.dropTableIfExists(database, schemaName, tmpTableName); + } + LOGGER.info("Cleaning tmp tables in destination completed."); + } + + ; } private static AirbyteStreamNameNamespacePair toNameNamespacePair(final WriteConfig config) { diff --git a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/StagingConsumerFactory.java b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/StagingConsumerFactory.java index 6f076bd95d787c..70c3a2b2eedc6a 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/StagingConsumerFactory.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/StagingConsumerFactory.java @@ -188,7 +188,6 @@ CheckedBiConsumer final StagingOperations stagingOperations, final List writeConfigs, final ConfiguredAirbyteCatalog catalog) { - // TODO: (ryankfu) move this block of code that executes before the lambda to #onStartFunction final Set conflictingStreams = new HashSet<>(); final Map pairToWriteConfig = new HashMap<>(); for (final WriteConfig config : writeConfigs) { diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile index fa7b464a6140f4..542f6ab8cdab0b 100644 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.23 +LABEL io.airbyte.version=0.1.22 LABEL io.airbyte.name=airbyte/destination-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-mssql/Dockerfile b/airbyte-integrations/connectors/destination-mssql/Dockerfile index 517102e715cb08..8141e99ddceabe 100644 --- a/airbyte-integrations/connectors/destination-mssql/Dockerfile +++ b/airbyte-integrations/connectors/destination-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.23 +LABEL io.airbyte.version=0.1.22 LABEL io.airbyte.name=airbyte/destination-mssql diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile index c5da82013d4c76..4962191e6f3da4 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.27 +LABEL io.airbyte.version=0.3.26 LABEL io.airbyte.name=airbyte/destination-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-postgres/Dockerfile b/airbyte-integrations/connectors/destination-postgres/Dockerfile index 63607b4570db4e..0ba0237ece8cc7 100644 --- a/airbyte-integrations/connectors/destination-postgres/Dockerfile +++ b/airbyte-integrations/connectors/destination-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.27 +LABEL io.airbyte.version=0.3.26 LABEL io.airbyte.name=airbyte/destination-postgres diff --git a/airbyte-integrations/connectors/destination-tidb/Dockerfile b/airbyte-integrations/connectors/destination-tidb/Dockerfile index 740ab614a65ea2..46a42427b19c41 100644 --- a/airbyte-integrations/connectors/destination-tidb/Dockerfile +++ b/airbyte-integrations/connectors/destination-tidb/Dockerfile @@ -14,5 +14,5 @@ ENV APPLICATION destination-tidb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.0 LABEL io.airbyte.name=airbyte/destination-tidb diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md index 4db25cdc5d7e32..f415af4fd10c0a 100644 --- a/docs/integrations/destinations/clickhouse.md +++ b/docs/integrations/destinations/clickhouse.md @@ -3,12 +3,12 @@ ## Features -| Feature | Supported?\(Yes/No\) | Notes | -|:------------------------------|:---------------------|:------| -| Full Refresh Sync | Yes | | -| Incremental - Append Sync | Yes | | -| Incremental - Deduped History | Yes | | -| Namespaces | Yes | | +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Incremental - Deduped History | Yes | | +| Namespaces | Yes | | #### Output Schema @@ -80,17 +80,16 @@ Therefore, Airbyte ClickHouse destination will create tables and schemas using t | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:--------------------------------------------------------------------| -| 0.2.3 | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing | -| 0.2.2 | 2023-02-21 | [\#21509](https://github.com/airbytehq/airbyte/pull/21509) | Compatibility update with security patch for strict encrypt version | -| 0.2.1 | 2022-12-06 | [\#19573](https://github.com/airbytehq/airbyte/pull/19573) | Update dbt version to 1.3.1 | -| 0.2.0 | 2022-09-27 | [\#16970](https://github.com/airbytehq/airbyte/pull/16970) | Remove TCP port from spec parameters | -| 0.1.12 | 2022-09-08 | [\#16444](https://github.com/airbytehq/airbyte/pull/16444) | Added custom jdbc params field | +| 0.2.2 | 2023-02-21 | [21509](https://github.com/airbytehq/airbyte/pull/21509) | Compatibility update with security patch for strict encrypt version | +| 0.2.1 | 2022-12-06 | [19573](https://github.com/airbytehq/airbyte/pull/19573) | Update dbt version to 1.3.1 | +| 0.2.0 | 2022-09-27 | [16970](https://github.com/airbytehq/airbyte/pull/16970) | Remove TCP port from spec parameters | +| 0.1.12 | 2022-09-08 | [16444](https://github.com/airbytehq/airbyte/pull/16444) | Added custom jdbc params field | | 0.1.10 | 2022-07-05 | [\#13639](https://github.com/airbytehq/airbyte/pull/13639) | Change JDBC ClickHouse version into 0.3.2-patch9 | | 0.1.8 | 2022-07-05 | [\#13516](https://github.com/airbytehq/airbyte/pull/13516) | Added JDBC default parameter socket timeout | | 0.1.7 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | -| 0.1.6 | 2022-05-17 | [\#12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | -| 0.1.5 | 2022-04-06 | [\#11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | -| 0.1.4 | 2022-02-25 | [\#10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | -| 0.1.3 | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.6 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | +| 0.1.5 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | +| 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.1 | 2021-12-21 | [\#8982](https://github.com/airbytehq/airbyte/pull/8982) | Set isSchemaRequired to false | | 0.1.0 | 2021-11-04 | [\#7620](https://github.com/airbytehq/airbyte/pull/7620) | Add ClickHouse destination | diff --git a/docs/integrations/destinations/mssql.md b/docs/integrations/destinations/mssql.md index 785d5b909b76c9..d2307042c1dce0 100644 --- a/docs/integrations/destinations/mssql.md +++ b/docs/integrations/destinations/mssql.md @@ -2,12 +2,12 @@ ## Features -| Feature | Supported?\(Yes/No\) | Notes | -|:------------------------------|:---------------------|:---------------------------------------------------------------------------------------------| -| Full Refresh Sync | Yes | | -| Incremental - Append Sync | Yes | | -| Incremental - Deduped History | No | As this connector does not support dbt, we don't support this sync mode on this destination. | -| Namespaces | Yes | | +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Incremental - Deduped History | No | As this connector does not support dbt, we don't support this sync mode on this destination. | +| Namespaces | Yes | | ## Output Schema @@ -26,12 +26,12 @@ Each stream will be output into its own table in SQL Server. Each table will con Airbyte Cloud only supports connecting to your MSSQL instance with TLS encryption. Other than that, you can proceed with the open-source instructions below. -| Feature | Supported?\(Yes/No\) | Notes | -|:------------------------------|:---------------------|:------| -| Full Refresh Sync | Yes | | -| Incremental - Append Sync | Yes | | -| Incremental - Deduped History | Yes | | -| Namespaces | Yes | | +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Incremental - Deduped History | Yes | | +| Namespaces | Yes | | ## Getting Started \(Airbyte Open-Source\) @@ -114,44 +114,42 @@ Using this feature requires additional configuration, when creating the source. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------| -| 0.1.23 | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing | -| 0.1.22 | 2022-10-21 | [\#18275](https://github.com/airbytehq/airbyte/pull/18275) | Upgrade commons-text for CVE 2022-42889 | -| 0.1.20 | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors | -| 0.1.19 | 2022-05-25 | [\#13054](https://github.com/airbytehq/airbyte/pull/13054) | Destination MSSQL: added custom JDBC parameters support. | -| 0.1.18 | 2022-05-17 | [\#12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | -| 0.1.17 | 2022-04-05 | [\#11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | -| 0.1.15 | 2022-02-25 | [\#10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | -| 0.1.14 | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.13 | 2021-12-28 | [\#9158](https://github.com/airbytehq/airbyte/pull/9158) | Update connector fields title/description | -| 0.1.12 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | -| 0.1.11 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | -| 0.1.10 | 2021-10-11 | [\#6877](https://github.com/airbytehq/airbyte/pull/6877) | Add `normalization` capability, add `append+deduplication` sync mode | -| 0.1.9 | 2021-09-29 | [\#5970](https://github.com/airbytehq/airbyte/pull/5970) | Add support & test cases for MSSQL Destination via SSH tunnels | -| 0.1.8 | 2021-08-07 | [\#5272](https://github.com/airbytehq/airbyte/pull/5272) | Add batch method to insert records | -| 0.1.7 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | -| 0.1.6 | 2021-06-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer | -| 0.1.5 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | declare object types correctly in spec | -| 0.1.4 | 2021-06-17 | [\#3744](https://github.com/airbytehq/airbyte/pull/3744) | Fix doc/params in specification file | -| 0.1.3 | 2021-05-28 | [\#3728](https://github.com/airbytehq/airbyte/pull/3973) | Change dockerfile entrypoint | -| 0.1.2 | 2021-05-13 | [\#3367](https://github.com/airbytehq/airbyte/pull/3671) | Fix handle symbols unicode | -| 0.1.1 | 2021-05-11 | [\#3566](https://github.com/airbytehq/airbyte/pull/3195) | MS SQL Server Destination Release! | +| Version | Date | Pull Request | Subject | +|:--------| :--- |:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------| +| 0.1.22 | 2022-10-21 | [18275](https://github.com/airbytehq/airbyte/pull/18275) | Upgrade commons-text for CVE 2022-42889 | +| 0.1.20 | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors | +| 0.1.19 | 2022-05-25 | [13054](https://github.com/airbytehq/airbyte/pull/13054) | Destination MSSQL: added custom JDBC parameters support. | +| 0.1.18 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | +| 0.1.17 | 2022-04-05 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | +| 0.1.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.1.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.13 | 2021-12-28 | [\#9158](https://github.com/airbytehq/airbyte/pull/9158) | Update connector fields title/description | +| 0.1.12 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | +| 0.1.11 | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | +| 0.1.10 | 2021-10-11 | [\#6877](https://github.com/airbytehq/airbyte/pull/6877) | Add `normalization` capability, add `append+deduplication` sync mode | +| 0.1.9 | 2021-09-29 | [\#5970](https://github.com/airbytehq/airbyte/pull/5970) | Add support & test cases for MSSQL Destination via SSH tunnels | +| 0.1.8 | 2021-08-07 | [\#5272](https://github.com/airbytehq/airbyte/pull/5272) | Add batch method to insert records | +| 0.1.7 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | +| 0.1.6 | 2021-06-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer | +| 0.1.5 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | declare object types correctly in spec | +| 0.1.4 | 2021-06-17 | [\#3744](https://github.com/airbytehq/airbyte/pull/3744) | Fix doc/params in specification file | +| 0.1.3 | 2021-05-28 | [\#3728](https://github.com/airbytehq/airbyte/pull/3973) | Change dockerfile entrypoint | +| 0.1.2 | 2021-05-13 | [\#3367](https://github.com/airbytehq/airbyte/pull/3671) | Fix handle symbols unicode | +| 0.1.1 | 2021-05-11 | [\#3566](https://github.com/airbytehq/airbyte/pull/3195) | MS SQL Server Destination Release! | ### Changelog (Strict Encrypt) -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------| -| 0.1.23 | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing | -| 0.1.22 | 2022-10-21 | [\#18275](https://github.com/airbytehq/airbyte/pull/18275) | Upgrade commons-text for CVE 2022-42889 | -| 0.1.21 | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage | -| 0.1.20 | 2022-07-14 | [\#15260](https://github.com/airbytehq/airbyte/pull/15260) | Align version of strict encrypt connector with regular connector | -| 0.1.10 | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors | -| 0.1.9 | 2022-06-17 | [\#13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | -| 0.1.8 | 2022-05-25 | [\#13054](https://github.com/airbytehq/airbyte/pull/13054) | Destination MSSQL: added custom JDBC parameters support. | -| 0.1.6 | 2022-05-17 | [\#12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | -| 0.1.5 | 2022-02-25 | [\#10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | -| 0.1.4 | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.3 | 2021-12-28 | [\#9158](https://github.com/airbytehq/airbyte/pull/9158) | Update connector fields title/description | -| 0.1.2 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | -| 0.1.1 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | +| Version | Date | Pull Request | Subject | +|:--------| :--- | :--- |:----------------------------------------------------------------------------------------------------| +| 0.1.22 | 2022-10-21 | [18275](https://github.com/airbytehq/airbyte/pull/18275) | Upgrade commons-text for CVE 2022-42889 | +| 0.1.21 | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage | +| 0.1.20 | 2022-07-14 | [\#15260](https://github.com/airbytehq/airbyte/pull/15260) | Align version of strict encrypt connector with regular connector | +| 0.1.10 | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors | +| 0.1.9 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | +| 0.1.8 | 2022-05-25 | [13054](https://github.com/airbytehq/airbyte/pull/13054) | Destination MSSQL: added custom JDBC parameters support. | +| 0.1.6 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | +| 0.1.5 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.1.4 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.3 | 2021-12-28 | [\#9158](https://github.com/airbytehq/airbyte/pull/9158) | Update connector fields title/description | +| 0.1.2 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | +| 0.1.1 | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | diff --git a/docs/integrations/destinations/postgres.md b/docs/integrations/destinations/postgres.md index 59879985602d44..acacec90e80bd8 100644 --- a/docs/integrations/destinations/postgres.md +++ b/docs/integrations/destinations/postgres.md @@ -132,12 +132,12 @@ characters. The Postgres destination connector supports the following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): -| Feature | Supported?\(Yes/No\) | Notes | -|:------------------------------|:---------------------|:------| -| Full Refresh Sync | Yes | | -| Incremental - Append Sync | Yes | | -| Incremental - Deduped History | Yes | | -| Namespaces | Yes | | +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Incremental - Deduped History | Yes | | +| Namespaces | Yes | | ## Schema map @@ -163,22 +163,21 @@ Now that you have set up the Postgres destination connector, check out the follo ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------| -| 0.3.27 | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing | -| 0.3.26 | 2022-09-27 | [\#17299](https://github.com/airbytehq/airbyte/pull/17299) | Improve error handling for strict-encrypt postgres destination | -| 0.3.24 | 2022-09-08 | [\#16046](https://github.com/airbytehq/airbyte/pull/16046) | Fix missing database name URL Encoding | -| 0.3.23 | 2022-07-18 | [\#16260](https://github.com/airbytehq/airbyte/pull/16260) | Prevent traffic going on an unsecured channel in strict-encryption version of destination postgres | -| 0.3.22 | 2022-07-18 | [\#13840](https://github.com/airbytehq/airbyte/pull/13840) | Added the ability to connect using different SSL modes and SSL certificates | -| 0.3.21 | 2022-07-06 | [\#14479](https://github.com/airbytehq/airbyte/pull/14479) | Publish amd64 and arm64 versions of the connector | -| 0.3.20 | 2022-05-17 | [\#12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | -| 0.3.19 | 2022-04-25 | [\#12195](https://github.com/airbytehq/airbyte/pull/12195) | Add support for additional JDBC URL Params input | -| 0.3.18 | 2022-04-12 | [\#11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0 | -| 0.3.17 | 2022-04-05 | [\#11729](https://github.com/airbytehq/airbyte/pull/11729) | Fixed bug with dashes in schema name | -| 0.3.15 | 2022-02-25 | [\#10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | -| 0.3.14 | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.3.13 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | -| 0.3.12 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | -| 0.3.11 | 2021-09-07 | [\#5743](https://github.com/airbytehq/airbyte/pull/5743) | Add SSH Tunnel support | -| 0.3.10 | 2021-08-11 | [\#5336](https://github.com/airbytehq/airbyte/pull/5336) | Destination Postgres: fix \u0000\(NULL\) value processing | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------| +| 0.3.26 | 2022-09-27 | [17299](https://github.com/airbytehq/airbyte/pull/17299) | Improve error handling for strict-encrypt postgres destination | +| 0.3.24 | 2022-09-08 | [16046](https://github.com/airbytehq/airbyte/pull/16046) | Fix missing database name URL Encoding | +| 0.3.23 | 2022-07-18 | [16260](https://github.com/airbytehq/airbyte/pull/16260) | Prevent traffic going on an unsecured channel in strict-encryption version of destination postgres | +| 0.3.22 | 2022-07-18 | [13840](https://github.com/airbytehq/airbyte/pull/13840) | Added the ability to connect using different SSL modes and SSL certificates | +| 0.3.21 | 2022-07-06 | [14479](https://github.com/airbytehq/airbyte/pull/14479) | Publish amd64 and arm64 versions of the connector | +| 0.3.20 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | +| 0.3.19 | 2022-04-25 | [12195](https://github.com/airbytehq/airbyte/pull/12195) | Add support for additional JDBC URL Params input | +| 0.3.18 | 2022-04-12 | [11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0 | +| 0.3.17 | 2022-04-05 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Fixed bug with dashes in schema name | +| 0.3.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.3.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.3.13 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | +| 0.3.12 | 2021-11-08 | [7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | +| 0.3.11 | 2021-09-07 | [5743](https://github.com/airbytehq/airbyte/pull/5743) | Add SSH Tunnel support | +| 0.3.10 | 2021-08-11 | [5336](https://github.com/airbytehq/airbyte/pull/5336) | Destination Postgres: fix \u0000\(NULL\) value processing | diff --git a/docs/integrations/destinations/tidb.md b/docs/integrations/destinations/tidb.md index c2f5b6f6215c68..e599897c30c0c4 100644 --- a/docs/integrations/destinations/tidb.md +++ b/docs/integrations/destinations/tidb.md @@ -97,7 +97,6 @@ Using this feature requires additional configuration, when creating the destinat ## CHANGELOG -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:-----------------------------------------------------------|:--------------------------------------| -| 0.1.1 | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing | -| 0.1.0 | 2022-08-12 | [\#15592](https://github.com/airbytehq/airbyte/pull/15592) | Added TiDB destination. | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:--------------------------------------------------------------|:------------------------| +| 0.1.0 | 2022-08-12 | [#15592](https://github.com/airbytehq/airbyte/pull/15592) | Added TiDB destination. |