From bfd05f6c0e77fceeeab844ae1684083799816d2b Mon Sep 17 00:00:00 2001 From: Andrii Leonets Date: Thu, 3 Jun 2021 15:55:50 +0300 Subject: [PATCH 1/4] implement additional handler for the Integer source values in order to cover the case with too large value (possible for some sources) --- .../main/java/io/airbyte/db/jdbc/JdbcUtils.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index 8a38883dbd7c2..a33bc258cdd97 100644 --- a/airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -132,7 +132,7 @@ private static void setJsonField(ResultSet r, int i, ObjectNode o) throws SQLExc switch (columnType) { case BIT, BOOLEAN -> o.put(columnName, r.getBoolean(i)); case TINYINT, SMALLINT -> o.put(columnName, r.getShort(i)); - case INTEGER -> o.put(columnName, r.getInt(i)); + case INTEGER -> putInteger(o, columnName, r, i); case BIGINT -> o.put(columnName, nullIfInvalid(() -> r.getLong(i))); case FLOAT, DOUBLE -> o.put(columnName, nullIfInvalid(() -> r.getDouble(i), Double::isFinite)); case REAL -> o.put(columnName, nullIfInvalid(() -> r.getFloat(i), Float::isFinite)); @@ -151,6 +151,19 @@ private static void setJsonField(ResultSet r, int i, ObjectNode o) throws SQLExc } } + /** + * In some sources Integer might have value larger than {@link Integer#MAX_VALUE}. E.q. MySQL has unsigned Integer type, + * which can contain value 3428724653. + * If we fail to cast Integer value, we will try to cast Long. + */ + private static void putInteger(ObjectNode node, String columnName, ResultSet r, int i) { + try { + node.put(columnName, r.getInt(i)); + } catch (SQLException e) { + node.put(columnName, nullIfInvalid(() -> r.getLong(i))); + } + } + // todo (cgardens) - move generic date helpers to commons. public static String toISO8601String(long epochMillis) { From a218b467ba9e01247c0763af2971753f2c815158 Mon Sep 17 00:00:00 2001 From: Andrii Leonets Date: Thu, 3 Jun 2021 15:58:19 +0300 Subject: [PATCH 2/4] format --- airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index a33bc258cdd97..b3b949fde3dfe 100644 --- a/airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -152,9 +152,9 @@ private static void setJsonField(ResultSet r, int i, ObjectNode o) throws SQLExc } /** - * In some sources Integer might have value larger than {@link Integer#MAX_VALUE}. E.q. MySQL has unsigned Integer type, - * which can contain value 3428724653. - * If we fail to cast Integer value, we will try to cast Long. + * In some sources Integer might have value larger than {@link Integer#MAX_VALUE}. E.q. MySQL has + * unsigned Integer type, which can contain value 3428724653. If we fail to cast Integer value, we + * will try to cast Long. */ private static void putInteger(ObjectNode node, String columnName, ResultSet r, int i) { try { From b321ecaae5c2dba332b23ec0000553c61827cb30 Mon Sep 17 00:00:00 2001 From: Andrii Leonets Date: Thu, 3 Jun 2021 18:16:41 +0300 Subject: [PATCH 3/4] [internal review] rename inputs --- airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index b3b949fde3dfe..ac6260b9add1d 100644 --- a/airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -156,11 +156,11 @@ private static void setJsonField(ResultSet r, int i, ObjectNode o) throws SQLExc * unsigned Integer type, which can contain value 3428724653. If we fail to cast Integer value, we * will try to cast Long. */ - private static void putInteger(ObjectNode node, String columnName, ResultSet r, int i) { + private static void putInteger(ObjectNode node, String columnName, ResultSet resultSet, int index) { try { - node.put(columnName, r.getInt(i)); + node.put(columnName, resultSet.getInt(index)); } catch (SQLException e) { - node.put(columnName, nullIfInvalid(() -> r.getLong(i))); + node.put(columnName, nullIfInvalid(() -> resultSet.getLong(index))); } } From 081cb067bf042a6794c1e1d3dae9cf7cbece5dcf Mon Sep 17 00:00:00 2001 From: Andrii Leonets Date: Fri, 4 Jun 2021 12:29:14 +0300 Subject: [PATCH 4/4] upgrade the MySQL Source docker version to 0.3.4 --- .../435bb9a5-7887-4809-aa58-28c27df0d7ad.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-mysql/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/435bb9a5-7887-4809-aa58-28c27df0d7ad.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/435bb9a5-7887-4809-aa58-28c27df0d7ad.json index 7a12538826c07..eacd8a73ad20b 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/435bb9a5-7887-4809-aa58-28c27df0d7ad.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/435bb9a5-7887-4809-aa58-28c27df0d7ad.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "435bb9a5-7887-4809-aa58-28c27df0d7ad", "name": "MySQL", "dockerRepository": "airbyte/source-mysql", - "dockerImageTag": "0.3.3", + "dockerImageTag": "0.3.4", "documentationUrl": "https://docs.airbyte.io/integrations/sources/mysql", "icon": "mysql.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 5f0403dea55ec..c0a04e837dfe6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -71,7 +71,7 @@ - sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad name: MySQL dockerRepository: airbyte/source-mysql - dockerImageTag: 0.3.3 + dockerImageTag: 0.3.4 documentationUrl: https://docs.airbyte.io/integrations/sources/mysql icon: mysql.svg - sourceDefinitionId: 2470e835-feaf-4db6-96f3-70fd645acc77 diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile index 72763700d6b18..d1dfaa75e26aa 100644 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql/Dockerfile @@ -8,6 +8,6 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.3 +LABEL io.airbyte.version=0.3.4 LABEL io.airbyte.name=airbyte/source-mysql