diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json index ff762f8c9f705..d4c34cd6efa70 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "b5ea17b1-f170-46dc-bc31-cc744ca984c1", "name": "Microsoft SQL Server (MSSQL)", "dockerRepository": "airbyte/source-mssql", - "dockerImageTag": "0.3.8", + "dockerImageTag": "0.3.9", "documentationUrl": "https://docs.airbyte.io/integrations/sources/mssql", "icon": "mssql.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 4d6f7994483af..46ce0b9aa4d23 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -326,7 +326,7 @@ - name: Microsoft SQL Server (MSSQL) sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 dockerRepository: airbyte/source-mssql - dockerImageTag: 0.3.6 + dockerImageTag: 0.3.9 documentationUrl: https://docs.airbyte.io/integrations/sources/mssql icon: mssql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 590114c86993e..7bb48978a10e6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3198,7 +3198,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mssql:0.3.6" +- dockerImage: "airbyte/source-mssql:0.3.9" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mssql" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java index c3b611d948ae3..db67612eabebf 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java +++ b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java @@ -263,7 +263,7 @@ public JdbcDatabase createDatabase(final JsonNode config) throws SQLException { } protected JdbcSourceOperations getSourceOperations() { - return JdbcUtils.getDefaultSourceOperations(); + return sourceOperations; } } diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile index adb13884e8aa4..98311b09446de 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile index b73c16413c579..683bad6db02ce 100644 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.8 +LABEL io.airbyte.version=0.3.9 LABEL io.airbyte.name=airbyte/source-mssql diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java index 2099dc6892bc8..7a06eedea9402 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java @@ -15,8 +15,6 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcSourceOperations; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.base.ssh.SshWrappedSource; @@ -55,15 +53,12 @@ public class MssqlSource extends AbstractJdbcSource implements Source { public static final List HOST_KEY = List.of("host"); public static final List PORT_KEY = List.of("port"); - private final JdbcSourceOperations sourceOperations; - public static Source sshWrappedSource() { return new SshWrappedSource(new MssqlSource(), HOST_KEY, PORT_KEY); } MssqlSource() { - super(DRIVER_CLASS, new MssqlJdbcStreamingQueryConfiguration()); - this.sourceOperations = JdbcUtils.getDefaultSourceOperations(); + super(DRIVER_CLASS, new MssqlJdbcStreamingQueryConfiguration(), new MssqlSourceOperations()); } @Override diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java new file mode 100644 index 0000000000000..17ca2d0c96e8e --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.db.jdbc.JdbcSourceOperations; +import java.sql.ResultSet; +import java.sql.SQLException; + +public class MssqlSourceOperations extends JdbcSourceOperations { + + protected void putBinary(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { + byte[] bytes = resultSet.getBytes(index); + String value = new String(bytes); + node.put(columnName, value); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java index e9b5aec239b37..796a7f2a6ff26 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java @@ -121,8 +121,6 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "0", "1", "'true'", "'false'") .addExpectedValues(null, "false", "true", "true", "false") - .addInsertValues("null") - .addNullExpectedValue() .build()); addDataTypeTestData( @@ -285,27 +283,21 @@ protected void initTests() { null, "\\xF0\\x9F\\x9A\\x80") .build()); - // TODO BUG Returns binary value instead of actual value addDataTypeTestData( TestDataHolder.builder() .sourceType("binary") .airbyteType(JsonSchemaPrimitive.STRING) - // .addInsertValues("CAST( 'A' AS VARBINARY)", "null") - // .addExpectedValues("A") - .addInsertValues("null") - .addNullExpectedValue() + .addInsertValues("CAST( 'A' AS BINARY(1))", "null") + .addExpectedValues("A", null) .build()); - // TODO BUG Returns binary value instead of actual value addDataTypeTestData( TestDataHolder.builder() .sourceType("varbinary") - .fullSourceDataType("varbinary(30)") + .fullSourceDataType("varbinary(3)") .airbyteType(JsonSchemaPrimitive.STRING) - // .addInsertValues("CAST( 'ABC' AS VARBINARY)", "null") - // .addExpectedValues("A") - .addInsertValues("null") - .addNullExpectedValue() + .addInsertValues("CAST( 'ABC' AS VARBINARY)", "null") + .addExpectedValues("ABC", null) .build()); // TODO BUG: airbyte returns binary representation instead of readable one diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java index 677472ff0ce5f..0ed372d7971e6 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java @@ -15,7 +15,6 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcSourceOperations; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.base.ssh.SshWrappedSource; @@ -254,9 +253,4 @@ public enum ReplicationMethod { CDC } - @Override - protected JdbcSourceOperations getSourceOperations() { - return new MySqlSourceOperations(); - } - } diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java index 2bd067641e4b6..f7d39b22dbe2e 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcSourceOperations; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; @@ -53,9 +52,4 @@ public Set getExcludedInternalNameSpaces() { "INFORMATION_SCHEMA"); } - @Override - protected JdbcSourceOperations getSourceOperations() { - return new SnowflakeSourceOperations(); - } - } diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index 35e3621541df3..6b088e2f6731a 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -294,6 +294,7 @@ If you do not see a type in this list, assume that it is coerced into a string. | Version | Date | Pull Request | Subject | | | :--- | :--- | :--- | :--- | :--- | +| 0.3.9 | 2021-11-09 | [7386](https://github.com/airbytehq/airbyte/pull/7386) | Improve support for binary and varbinary data types | | | 0.3.8 | 2021-10-26 | [7386](https://github.com/airbytehq/airbyte/pull/7386) | Fixed data type (smalldatetime, smallmoney) conversion from mssql source | | | 0.3.7 | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585) | Improved SSH Tunnel key generation steps | | | 0.3.6 | 2021-09-17 | [6318](https://github.com/airbytehq/airbyte/pull/6318) | Added option to connect to DB via SSH | |