🐛 Fix handling binary data type for Mssql source (#7748)

* Fix data type tests Mssql source * update test for boolean data type in Mysql source * dump versions for Mssql and Mysql source * revert changes regarding handling bit data type * revert changes regarding date and time data type tests Mssql source * revert changes regarding date data type for Mssql source * update message in mssql.md * minor format changes * format changes
airbytehq · Nov 11, 2021 · 70ed18b · 70ed18b
1 parent bea9bb5
commit 70ed18b
Show file tree

Hide file tree

Showing 12 changed files with 33 additions and 37 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json
@@ -2,7 +2,7 @@
   "sourceDefinitionId": "b5ea17b1-f170-46dc-bc31-cc744ca984c1",
   "name": "Microsoft SQL Server (MSSQL)",
   "dockerRepository": "airbyte/source-mssql",
-  "dockerImageTag": "0.3.8",
+  "dockerImageTag": "0.3.9",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/mssql",
   "icon": "mssql.svg"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -326,7 +326,7 @@
 - name: Microsoft SQL Server (MSSQL)
   sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1
   dockerRepository: airbyte/source-mssql
-  dockerImageTag: 0.3.6
+  dockerImageTag: 0.3.9
   documentationUrl: https://docs.airbyte.io/integrations/sources/mssql
   icon: mssql.svg
   sourceType: database

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -3198,7 +3198,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-mssql:0.3.6"
+- dockerImage: "airbyte/source-mssql:0.3.9"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/mssql"
     connectionSpecification:

diff --git a/...ors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/...ors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java
@@ -263,7 +263,7 @@ public JdbcDatabase createDatabase(final JsonNode config) throws SQLException {
   }
 
   protected JdbcSourceOperations getSourceOperations() {
-    return JdbcUtils.getDefaultSourceOperations();
+    return sourceOperations;
   }
 
 }
diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile
@@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 
-LABEL io.airbyte.version=0.1.1
+LABEL io.airbyte.version=0.1.2
 LABEL io.airbyte.name=airbyte/source-mssql-strict-encrypt
diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile
@@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 
-LABEL io.airbyte.version=0.3.8
+LABEL io.airbyte.version=0.3.9
 LABEL io.airbyte.name=airbyte/source-mssql
diff --git a/...nnectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java b/...nnectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java
@@ -15,8 +15,6 @@
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.commons.util.AutoCloseableIterator;
 import io.airbyte.db.jdbc.JdbcDatabase;
-import io.airbyte.db.jdbc.JdbcSourceOperations;
-import io.airbyte.db.jdbc.JdbcUtils;
 import io.airbyte.integrations.base.IntegrationRunner;
 import io.airbyte.integrations.base.Source;
 import io.airbyte.integrations.base.ssh.SshWrappedSource;
@@ -55,15 +53,12 @@ public class MssqlSource extends AbstractJdbcSource implements Source {
   public static final List<String> HOST_KEY = List.of("host");
   public static final List<String> PORT_KEY = List.of("port");
 
-  private final JdbcSourceOperations sourceOperations;
-
   public static Source sshWrappedSource() {
     return new SshWrappedSource(new MssqlSource(), HOST_KEY, PORT_KEY);
   }
 
   MssqlSource() {
-    super(DRIVER_CLASS, new MssqlJdbcStreamingQueryConfiguration());
-    this.sourceOperations = JdbcUtils.getDefaultSourceOperations();
+    super(DRIVER_CLASS, new MssqlJdbcStreamingQueryConfiguration(), new MssqlSourceOperations());
   }
 
   @Override

diff --git a/...ource-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java b/...ource-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.integrations.source.mssql;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.airbyte.db.jdbc.JdbcSourceOperations;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class MssqlSourceOperations extends JdbcSourceOperations {
+
+  protected void putBinary(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException {
+    byte[] bytes = resultSet.getBytes(index);
+    String value = new String(bytes);
+    node.put(columnName, value);
+  }
+
+}
diff --git a/...c/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java b/...c/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java
@@ -121,8 +121,6 @@ protected void initTests() {
             .airbyteType(JsonSchemaPrimitive.NUMBER)
             .addInsertValues("null", "0", "1", "'true'", "'false'")
             .addExpectedValues(null, "false", "true", "true", "false")
-            .addInsertValues("null")
-            .addNullExpectedValue()
             .build());
 
     addDataTypeTestData(
@@ -285,27 +283,21 @@ protected void initTests() {
                 null, "\\xF0\\x9F\\x9A\\x80")
             .build());
 
-    // TODO BUG Returns binary value instead of actual value
     addDataTypeTestData(
         TestDataHolder.builder()
             .sourceType("binary")
             .airbyteType(JsonSchemaPrimitive.STRING)
-            // .addInsertValues("CAST( 'A' AS VARBINARY)", "null")
-            // .addExpectedValues("A")
-            .addInsertValues("null")
-            .addNullExpectedValue()
+            .addInsertValues("CAST( 'A' AS BINARY(1))", "null")
+            .addExpectedValues("A", null)
             .build());
 
-    // TODO BUG Returns binary value instead of actual value
     addDataTypeTestData(
         TestDataHolder.builder()
             .sourceType("varbinary")
-            .fullSourceDataType("varbinary(30)")
+            .fullSourceDataType("varbinary(3)")
             .airbyteType(JsonSchemaPrimitive.STRING)
-            // .addInsertValues("CAST( 'ABC' AS VARBINARY)", "null")
-            // .addExpectedValues("A")
-            .addInsertValues("null")
-            .addNullExpectedValue()
+            .addInsertValues("CAST( 'ABC' AS VARBINARY)", "null")
+            .addExpectedValues("ABC", null)
             .build());
 
     // TODO BUG: airbyte returns binary representation instead of readable one

diff --git a/...nnectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java b/...nnectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java
@@ -15,7 +15,6 @@
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.commons.util.AutoCloseableIterator;
 import io.airbyte.db.jdbc.JdbcDatabase;
-import io.airbyte.db.jdbc.JdbcSourceOperations;
 import io.airbyte.integrations.base.IntegrationRunner;
 import io.airbyte.integrations.base.Source;
 import io.airbyte.integrations.base.ssh.SshWrappedSource;
@@ -254,9 +253,4 @@ public enum ReplicationMethod {
     CDC
   }
 
-  @Override
-  protected JdbcSourceOperations getSourceOperations() {
-    return new MySqlSourceOperations();
-  }
-
 }
diff --git a/...rce-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java b/...rce-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java
@@ -7,7 +7,6 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.collect.ImmutableMap;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.db.jdbc.JdbcSourceOperations;
 import io.airbyte.integrations.base.IntegrationRunner;
 import io.airbyte.integrations.base.Source;
 import io.airbyte.integrations.source.jdbc.AbstractJdbcSource;
@@ -53,9 +52,4 @@ public Set<String> getExcludedInternalNameSpaces() {
         "INFORMATION_SCHEMA");
   }
 
-  @Override
-  protected JdbcSourceOperations getSourceOperations() {
-    return new SnowflakeSourceOperations();
-  }
-
 }
diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md
@@ -294,6 +294,7 @@ If you do not see a type in this list, assume that it is coerced into a string.
 
 | Version | Date | Pull Request | Subject |  |
 | :--- | :--- | :--- | :--- | :--- |
+| 0.3.9 | 2021-11-09 | [7386](https://github.com/airbytehq/airbyte/pull/7386) | Improve support for binary and varbinary data types  |  |
 | 0.3.8 | 2021-10-26 | [7386](https://github.com/airbytehq/airbyte/pull/7386) | Fixed data type (smalldatetime, smallmoney) conversion from mssql source |  |
 | 0.3.7 | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585) | Improved SSH Tunnel key generation steps |  |
 | 0.3.6 | 2021-09-17 | [6318](https://github.com/airbytehq/airbyte/pull/6318) | Added option to connect to DB via SSH |  |