From 2bb869c82b9d128e443b76b90a09b819a6eb0316 Mon Sep 17 00:00:00 2001 From: Akash Kulkarni Date: Thu, 9 May 2024 15:57:04 -0700 Subject: [PATCH 1/9] Initial commit --- .../airbyte/cdk/integrations/util/ConnectorExceptionUtil.kt | 6 ++++++ airbyte-integrations/connectors/source-mysql/build.gradle | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/util/ConnectorExceptionUtil.kt b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/util/ConnectorExceptionUtil.kt index de0e3a49bfeb..bfbd6db65b04 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/util/ConnectorExceptionUtil.kt +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/util/ConnectorExceptionUtil.kt @@ -9,6 +9,7 @@ import io.airbyte.commons.exceptions.ConfigErrorException import io.airbyte.commons.exceptions.ConnectionErrorException import io.airbyte.commons.exceptions.TransientErrorException import io.airbyte.commons.functional.Either +import java.io.EOFException import java.sql.SQLException import java.sql.SQLSyntaxErrorException import java.util.stream.Collectors @@ -137,6 +138,11 @@ object ConnectorExceptionUtil { return e is ConnectionErrorException } + private fun isEOFException(e: Throwable?): Boolean { + return (e is EOFException) && + e.message!!.lowercase().contains("connection was unexpectedly lost"); + } + private fun isRecoveryConnectionException(e: Throwable?): Boolean { return e is SQLException && e.message!!.lowercase().contains("due to conflict with recovery") diff --git a/airbyte-integrations/connectors/source-mysql/build.gradle b/airbyte-integrations/connectors/source-mysql/build.gradle index 52936ba1217d..3ac889f1274d 100644 --- a/airbyte-integrations/connectors/source-mysql/build.gradle +++ b/airbyte-integrations/connectors/source-mysql/build.gradle @@ -8,7 +8,7 @@ plugins { airbyteJavaConnector { cdkVersionRequired = '0.33.1' features = ['db-sources'] - useLocalCdk = false + useLocalCdk = true } java { From 4dc984f530a3396bd2250490e282379056f97033 Mon Sep 17 00:00:00 2001 From: Akash Kulkarni Date: Fri, 10 May 2024 09:18:03 -0700 Subject: [PATCH 2/9] updates --- .../util/ConnectorExceptionUtil.kt | 18 +++++++++++++++--- .../connectors/source-mysql/metadata.yaml | 2 +- docs/integrations/sources/mysql.md | 1 + 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/util/ConnectorExceptionUtil.kt b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/util/ConnectorExceptionUtil.kt index bfbd6db65b04..7705ae54533e 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/util/ConnectorExceptionUtil.kt +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/util/ConnectorExceptionUtil.kt @@ -26,6 +26,8 @@ object ConnectorExceptionUtil { const val RECOVERY_CONNECTION_ERROR_MESSAGE: String = "We're having issues syncing from a Postgres replica that is configured as a hot standby server. " + "Please see https://go.airbyte.com/pg-hot-standby-error-message for options and workarounds" + const val DATABASE_CONNECTION_ERROR: String = + "Encountered an error while connecting to the database error" @JvmField val HTTP_AUTHENTICATION_ERROR_CODES: List = ImmutableList.of(401, 403) @@ -36,7 +38,10 @@ object ConnectorExceptionUtil { } fun isTransientError(e: Throwable?): Boolean { - return isTransientErrorException(e) || isRecoveryConnectionException(e) + return isTransientErrorException(e) || + isRecoveryConnectionException(e) || + isTransientEOFException(e) || + isTransientSQLException(e) } fun getDisplayMessage(e: Throwable?): String? { @@ -50,6 +55,8 @@ object ConnectorExceptionUtil { RECOVERY_CONNECTION_ERROR_MESSAGE } else if (isUnknownColumnInFieldListException(e)) { e!!.message + } else if (isTransientError(e)) { + DATABASE_CONNECTION_ERROR } else { String.format( COMMON_EXCEPTION_MESSAGE_TEMPLATE, @@ -138,9 +145,14 @@ object ConnectorExceptionUtil { return e is ConnectionErrorException } - private fun isEOFException(e: Throwable?): Boolean { + private fun isTransientEOFException(e: Throwable?): Boolean { return (e is EOFException) && - e.message!!.lowercase().contains("connection was unexpectedly lost"); + e.message!!.lowercase().contains("connection was unexpectedly lost") + } + + private fun isTransientSQLException(e: Throwable?): Boolean { + return (e is SQLException) && + e.message!!.lowercase().contains("An I/O error occurred while sending to the backend") } private fun isRecoveryConnectionException(e: Throwable?): Boolean { diff --git a/airbyte-integrations/connectors/source-mysql/metadata.yaml b/airbyte-integrations/connectors/source-mysql/metadata.yaml index b80b00d77137..d006c8fbe85b 100644 --- a/airbyte-integrations/connectors/source-mysql/metadata.yaml +++ b/airbyte-integrations/connectors/source-mysql/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad - dockerImageTag: 3.4.2 + dockerImageTag: 3.4.1 dockerRepository: airbyte/source-mysql documentationUrl: https://docs.airbyte.com/integrations/sources/mysql githubIssueLabel: source-mysql diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index 3ec66f8ab4ac..709cfde3074e 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -230,6 +230,7 @@ Any database or table encoding combination of charset and collation is supported | Version | Date | Pull Request | Subject | |:--------|:-----------| :--------------------------------------------------------- |:------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.4.3 | 2024-05-13 | [38104](https://github.com/airbytehq/airbyte/pull/38104) | Handle transient error messages. | | 3.4.2 | 2024-05-07 | [38046](https://github.com/airbytehq/airbyte/pull/38046) | Resumeable refresh should run only if there is source defined pk. | | 3.4.1 | 2024-05-03 | [37824](https://github.com/airbytehq/airbyte/pull/37824) | Fixed a bug on Resumeable full refresh where cursor based source throw NPE. | | 3.4.0 | 2024-05-02 | [36932](https://github.com/airbytehq/airbyte/pull/36932) | Resumeable full refresh. Note please upgrade your platform - minimum platform version is 0.58.0. | From 8806f47bcef7e2682dd9ced8a12582f9f4074fc9 Mon Sep 17 00:00:00 2001 From: Akash Kulkarni Date: Fri, 10 May 2024 09:20:57 -0700 Subject: [PATCH 3/9] bump metadata --- airbyte-integrations/connectors/source-mysql/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-mysql/metadata.yaml b/airbyte-integrations/connectors/source-mysql/metadata.yaml index d006c8fbe85b..aed0c5127eab 100644 --- a/airbyte-integrations/connectors/source-mysql/metadata.yaml +++ b/airbyte-integrations/connectors/source-mysql/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad - dockerImageTag: 3.4.1 + dockerImageTag: 3.4.3 dockerRepository: airbyte/source-mysql documentationUrl: https://docs.airbyte.com/integrations/sources/mysql githubIssueLabel: source-mysql From bada72ee1deb97f9e3561ef747f566286944b8c4 Mon Sep 17 00:00:00 2001 From: Akash Kulkarni Date: Mon, 13 May 2024 14:58:48 -0700 Subject: [PATCH 4/9] Bump pg too --- airbyte-integrations/connectors/source-postgres/build.gradle | 2 +- airbyte-integrations/connectors/source-postgres/metadata.yaml | 2 +- docs/integrations/sources/postgres.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index 3e21444195c4..019c7f50d701 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -14,7 +14,7 @@ java { airbyteJavaConnector { cdkVersionRequired = '0.34.2' features = ['db-sources', 'datastore-postgres'] - useLocalCdk = false + useLocalCdk = true } application { diff --git a/airbyte-integrations/connectors/source-postgres/metadata.yaml b/airbyte-integrations/connectors/source-postgres/metadata.yaml index 585ae0f47fa4..ae000fa70d4a 100644 --- a/airbyte-integrations/connectors/source-postgres/metadata.yaml +++ b/airbyte-integrations/connectors/source-postgres/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 - dockerImageTag: 3.4.1 + dockerImageTag: 3.4.2 dockerRepository: airbyte/source-postgres documentationUrl: https://docs.airbyte.com/integrations/sources/postgres githubIssueLabel: source-postgres diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index b96981e97b0b..55be8a21d36d 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -308,6 +308,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.4.2 | 2024-05-13 | [38104](https://github.com/airbytehq/airbyte/pull/38104) | Handle transient error messages. | | 3.4.1 | 2024-05-10 | [38130](https://github.com/airbytehq/airbyte/pull/38130) | Bug fix on old PG where ctid column not found when stream is a view. | | 3.4.0 | 2024-04-29 | [37112](https://github.com/airbytehq/airbyte/pull/37112) | resumeable full refresh. | | 3.3.33 | 2024-05-07 | [38030](https://github.com/airbytehq/airbyte/pull/38030) | Mark PG hot standby error as transient. | From 0359a5b1c4d1c8a3f635bb3e14ddef027d6e5ed9 Mon Sep 17 00:00:00 2001 From: Akash Kulkarni Date: Mon, 13 May 2024 15:39:40 -0700 Subject: [PATCH 5/9] fix build --- .../integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java | 1 + .../source/mysql/MySqlSslJdbcSourceAcceptanceTest.java | 1 + 2 files changed, 2 insertions(+) diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java index 3980f0fce249..742994efb05d 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java @@ -59,6 +59,7 @@ import org.junit.jupiter.api.Test; @Order(2) +@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "NP_NULL_ON_SOME_PATH") class MySqlJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { protected static final String USERNAME_WITHOUT_PERMISSION = "new_user"; diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java index 5d5ac314a928..bf47dd4da9bb 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java @@ -9,6 +9,7 @@ import org.junit.jupiter.api.Order; @Order(3) +@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "NP_NULL_ON_SOME_PATH") class MySqlSslJdbcSourceAcceptanceTest extends MySqlJdbcSourceAcceptanceTest { @Override From accfd0a43873c1eef4a5b8c441bd3b8667925c8b Mon Sep 17 00:00:00 2001 From: Akash Kulkarni Date: Mon, 13 May 2024 16:08:10 -0700 Subject: [PATCH 6/9] Bump cdk --- airbyte-cdk/java/airbyte-cdk/README.md | 1 + .../java/airbyte-cdk/core/src/main/resources/version.properties | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index ca4d236d3deb..a1e1436be077 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -174,6 +174,7 @@ corresponds to that version. | Version | Date | Pull Request | Subject | |:--------| :--------- | :--------------------------------------------------------- |:---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.35.1 | 2024-05-13 | [\#38104](https://github.com/airbytehq/airbyte/pull/38104) | Handle transient error messages | | 0.35.0 | 2024-05-13 | [\#38127](https://github.com/airbytehq/airbyte/pull/38127) | Destinations: Populate generation/sync ID on StreamConfig | | 0.34.4 | 2024-05-10 | [\#37712](https://github.com/airbytehq/airbyte/pull/37712) | make sure the exceptionHandler always terminates | | 0.34.4 | 2024-05-10 | [\#37712](https://github.com/airbytehq/airbyte/pull/37712) | make sure the exceptionHandler always terminates | diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index b621a5d5db2a..a6f110a58288 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.35.0 +version=0.35.1 From 0a859bed77390c875a27c7964d2fbbff7acd62ff Mon Sep 17 00:00:00 2001 From: Akash Kulkarni Date: Mon, 13 May 2024 16:15:23 -0700 Subject: [PATCH 7/9] toggle cdk --- airbyte-integrations/connectors/source-mysql/build.gradle | 4 ++-- airbyte-integrations/connectors/source-postgres/build.gradle | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-mysql/build.gradle b/airbyte-integrations/connectors/source-mysql/build.gradle index 3ac889f1274d..254ac5b1c0bb 100644 --- a/airbyte-integrations/connectors/source-mysql/build.gradle +++ b/airbyte-integrations/connectors/source-mysql/build.gradle @@ -6,9 +6,9 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.33.1' + cdkVersionRequired = '0.35.1' features = ['db-sources'] - useLocalCdk = true + useLocalCdk = false } java { diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index 4f1fc8cce016..678727bdd830 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -14,7 +14,7 @@ java { airbyteJavaConnector { cdkVersionRequired = '0.35.1' features = ['db-sources', 'datastore-postgres'] - useLocalCdk = true + useLocalCdk = false } application { From ece1f14f01984bc937fac38e3a16aed48e0ec7fd Mon Sep 17 00:00:00 2001 From: Akash Kulkarni Date: Mon, 13 May 2024 16:20:31 -0700 Subject: [PATCH 8/9] Fix metadata --- airbyte-cdk/java/airbyte-cdk/README.md | 2 +- .../airbyte-cdk/core/src/main/resources/version.properties | 2 +- airbyte-integrations/connectors/source-mysql/build.gradle | 4 ++-- airbyte-integrations/connectors/source-postgres/build.gradle | 4 ++-- airbyte-integrations/connectors/source-postgres/metadata.yaml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index a1e1436be077..d6f00ce00486 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -174,7 +174,7 @@ corresponds to that version. | Version | Date | Pull Request | Subject | |:--------| :--------- | :--------------------------------------------------------- |:---------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.35.1 | 2024-05-13 | [\#38104](https://github.com/airbytehq/airbyte/pull/38104) | Handle transient error messages | +| 0.35.2 | 2024-05-13 | [\#38104](https://github.com/airbytehq/airbyte/pull/38104) | Handle transient error messages | | 0.35.0 | 2024-05-13 | [\#38127](https://github.com/airbytehq/airbyte/pull/38127) | Destinations: Populate generation/sync ID on StreamConfig | | 0.34.4 | 2024-05-10 | [\#37712](https://github.com/airbytehq/airbyte/pull/37712) | make sure the exceptionHandler always terminates | | 0.34.4 | 2024-05-10 | [\#37712](https://github.com/airbytehq/airbyte/pull/37712) | make sure the exceptionHandler always terminates | diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index a6f110a58288..026e6f4179ea 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.35.1 +version=0.35.2 diff --git a/airbyte-integrations/connectors/source-mysql/build.gradle b/airbyte-integrations/connectors/source-mysql/build.gradle index 254ac5b1c0bb..e3b27576e45c 100644 --- a/airbyte-integrations/connectors/source-mysql/build.gradle +++ b/airbyte-integrations/connectors/source-mysql/build.gradle @@ -6,9 +6,9 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.35.1' + cdkVersionRequired = '0.35.2' features = ['db-sources'] - useLocalCdk = false + useLocalCdk = true } java { diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index 678727bdd830..6d1a59424f9f 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -12,9 +12,9 @@ java { } airbyteJavaConnector { - cdkVersionRequired = '0.35.1' + cdkVersionRequired = '0.35.2' features = ['db-sources', 'datastore-postgres'] - useLocalCdk = false + useLocalCdk = true } application { diff --git a/airbyte-integrations/connectors/source-postgres/metadata.yaml b/airbyte-integrations/connectors/source-postgres/metadata.yaml index ae000fa70d4a..886d1a949d4d 100644 --- a/airbyte-integrations/connectors/source-postgres/metadata.yaml +++ b/airbyte-integrations/connectors/source-postgres/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 - dockerImageTag: 3.4.2 + dockerImageTag: 3.4.3 dockerRepository: airbyte/source-postgres documentationUrl: https://docs.airbyte.com/integrations/sources/postgres githubIssueLabel: source-postgres From d1e0afdb002f8980eba2bf8af5f2420d534866b8 Mon Sep 17 00:00:00 2001 From: Akash Kulkarni Date: Mon, 13 May 2024 17:03:35 -0700 Subject: [PATCH 9/9] toggle useLocalCdk --- airbyte-integrations/connectors/source-mysql/build.gradle | 2 +- airbyte-integrations/connectors/source-postgres/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-mysql/build.gradle b/airbyte-integrations/connectors/source-mysql/build.gradle index e3b27576e45c..f1b53ab3a2da 100644 --- a/airbyte-integrations/connectors/source-mysql/build.gradle +++ b/airbyte-integrations/connectors/source-mysql/build.gradle @@ -8,7 +8,7 @@ plugins { airbyteJavaConnector { cdkVersionRequired = '0.35.2' features = ['db-sources'] - useLocalCdk = true + useLocalCdk = false } java { diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index 6d1a59424f9f..ffe8a50a8901 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -14,7 +14,7 @@ java { airbyteJavaConnector { cdkVersionRequired = '0.35.2' features = ['db-sources', 'datastore-postgres'] - useLocalCdk = true + useLocalCdk = false } application {