Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DB source errors] : Handle common transient errors #38104

Merged
merged 12 commits into from
May 14, 2024
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import io.airbyte.commons.exceptions.ConfigErrorException
import io.airbyte.commons.exceptions.ConnectionErrorException
import io.airbyte.commons.exceptions.TransientErrorException
import io.airbyte.commons.functional.Either
import java.io.EOFException
import java.sql.SQLException
import java.sql.SQLSyntaxErrorException
import java.util.stream.Collectors
Expand All @@ -25,6 +26,8 @@ object ConnectorExceptionUtil {
const val RECOVERY_CONNECTION_ERROR_MESSAGE: String =
"We're having issues syncing from a Postgres replica that is configured as a hot standby server. " +
"Please see https://go.airbyte.com/pg-hot-standby-error-message for options and workarounds"
const val DATABASE_CONNECTION_ERROR: String =
"Encountered an error while connecting to the database error"

@JvmField val HTTP_AUTHENTICATION_ERROR_CODES: List<Int> = ImmutableList.of(401, 403)

Expand All @@ -35,7 +38,10 @@ object ConnectorExceptionUtil {
}

fun isTransientError(e: Throwable?): Boolean {
return isTransientErrorException(e) || isRecoveryConnectionException(e)
return isTransientErrorException(e) ||
isRecoveryConnectionException(e) ||
isTransientEOFException(e) ||
isTransientSQLException(e)
}

fun getDisplayMessage(e: Throwable?): String? {
Expand All @@ -49,6 +55,8 @@ object ConnectorExceptionUtil {
RECOVERY_CONNECTION_ERROR_MESSAGE
} else if (isUnknownColumnInFieldListException(e)) {
e!!.message
} else if (isTransientError(e)) {
DATABASE_CONNECTION_ERROR
akashkulk marked this conversation as resolved.
Show resolved Hide resolved
} else {
String.format(
COMMON_EXCEPTION_MESSAGE_TEMPLATE,
Expand Down Expand Up @@ -137,6 +145,16 @@ object ConnectorExceptionUtil {
return e is ConnectionErrorException
}

private fun isTransientEOFException(e: Throwable?): Boolean {
return (e is EOFException) &&
e.message!!.lowercase().contains("connection was unexpectedly lost")
akashkulk marked this conversation as resolved.
Show resolved Hide resolved
}

private fun isTransientSQLException(e: Throwable?): Boolean {
return (e is SQLException) &&
e.message!!.lowercase().contains("An I/O error occurred while sending to the backend")
}

private fun isRecoveryConnectionException(e: Throwable?): Boolean {
return e is SQLException &&
e.message!!.lowercase().contains("due to conflict with recovery")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ plugins {
airbyteJavaConnector {
cdkVersionRequired = '0.33.1'
features = ['db-sources']
useLocalCdk = false
useLocalCdk = true
}

java {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ data:
connectorSubtype: database
connectorType: source
definitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad
dockerImageTag: 3.4.2
dockerImageTag: 3.4.3
dockerRepository: airbyte/source-mysql
documentationUrl: https://docs.airbyte.com/integrations/sources/mysql
githubIssueLabel: source-mysql
Expand Down
1 change: 1 addition & 0 deletions docs/integrations/sources/mysql.md
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,7 @@ Any database or table encoding combination of charset and collation is supported

| Version | Date | Pull Request | Subject |
|:--------|:-----------| :--------------------------------------------------------- |:------------------------------------------------------------------------------------------------------------------------------------------------|
| 3.4.3 | 2024-05-13 | [38104](https://github.com/airbytehq/airbyte/pull/38104) | Handle transient error messages. |
| 3.4.2 | 2024-05-07 | [38046](https://github.com/airbytehq/airbyte/pull/38046) | Resumeable refresh should run only if there is source defined pk. |
| 3.4.1 | 2024-05-03 | [37824](https://github.com/airbytehq/airbyte/pull/37824) | Fixed a bug on Resumeable full refresh where cursor based source throw NPE. |
| 3.4.0 | 2024-05-02 | [36932](https://github.com/airbytehq/airbyte/pull/36932) | Resumeable full refresh. Note please upgrade your platform - minimum platform version is 0.58.0. |
Expand Down