Skip to content

Commit

Permalink
Fix for Bulkcopy multi byte chars in char/varchar columns (#1671)
Browse files Browse the repository at this point in the history
  • Loading branch information
@lilgreenbird
lilgreenbird committed Nov 4, 2021
1 parent c135b63 commit 4a46e6c
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 25 deletions.
34 changes: 22 additions & 12 deletions src/main/java/com/microsoft/sqlserver/jdbc/IOBuffer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4018,9 +4018,7 @@ void writeStream(InputStream inputStream, long advertisedLength,
* efficiency. As this method will only be used in bulk copy, it needs to be efficient. Note: Any changes in
* algorithm/logic should propagate to both writeReader() and writeNonUnicodeReader().
*/

void writeNonUnicodeReader(Reader reader, long advertisedLength, boolean isDestBinary,
Charset charSet) throws SQLServerException {
void writeNonUnicodeReader(Reader reader, long advertisedLength, boolean isDestBinary) throws SQLServerException {
assert DataTypes.UNKNOWN_STREAM_LENGTH == advertisedLength || advertisedLength >= 0;

long actualLength = 0;
Expand Down Expand Up @@ -4057,18 +4055,30 @@ void writeNonUnicodeReader(Reader reader, long advertisedLength, boolean isDestB
// The Do-While loop goes on one more time as charsToWrite is greater than 0 for the last chunk, and
// in this last round the only thing that is written is an int value of 0, which is the PLP Terminator
// token(0x00000000).
writeInt(charsToWrite);

for (int charsCopied = 0; charsCopied < charsToWrite; ++charsCopied) {
if (null == charSet) {
// collation from database is the collation used
Charset charSet = con.getDatabaseCollation().getCharset();

if (null == charSet) {
writeInt(charsToWrite);

for (int charsCopied = 0; charsCopied < charsToWrite; ++charsCopied) {
streamByteBuffer[charsCopied] = (byte) (streamCharBuffer[charsCopied] & 0xFF);
} else {
// encoding as per collation
streamByteBuffer[charsCopied] = new String(streamCharBuffer[charsCopied] + "")
.getBytes(charSet)[0];
}

writeBytes(streamByteBuffer, 0, charsToWrite);
} else {
bytesToWrite = 0;
byte[] charBytes;
for (int charsCopied = 0; charsCopied < charsToWrite; ++charsCopied) {
charBytes = new String(streamCharBuffer[charsCopied] + "").getBytes(charSet);
System.arraycopy(charBytes, 0, streamByteBuffer, bytesToWrite, charBytes.length);
bytesToWrite += charBytes.length;
}

writeInt(bytesToWrite);
writeBytes(streamByteBuffer, 0, bytesToWrite);
}
writeBytes(streamByteBuffer, 0, charsToWrite);
} else {
bytesToWrite = charsToWrite;
if (0 != charsToWrite)
Expand Down Expand Up @@ -4982,7 +4992,7 @@ private void writeInternalTVPRowValues(JDBCType jdbcType, String currentColumnSt
// Null header for v*max types is 0xFFFFFFFFFFFFFFFF.
writeLong(0xFFFFFFFFFFFFFFFFL);
} else if (isSqlVariant) {
// for now we send as bigger type, but is sendStringParameterAsUnicoe is set to false we can't
// for now we send as bigger type, but is sendStringParameterAsUnicode is set to false we can't
// send nvarchar
// since we are writing as nvarchar we need to write as tdstype.bigvarchar value because if we
// want to supprot varchar(8000) it becomes as nvarchar, 8000*2 therefore we should send as
Expand Down
11 changes: 6 additions & 5 deletions src/main/java/com/microsoft/sqlserver/jdbc/SQLServerBulkCopy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2263,22 +2263,22 @@ else if (null != sourceCryptoMeta) {
} else {
reader = new StringReader(colValue.toString());
}

if (unicodeConversionRequired(bulkJdbcType, destSSType)) {
// writeReader is unicode.
tdsWriter.writeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH, true);
} else {
if ((SSType.BINARY == destSSType) || (SSType.VARBINARY == destSSType)
|| (SSType.VARBINARYMAX == destSSType) || (SSType.IMAGE == destSSType)) {
tdsWriter.writeNonUnicodeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH, true,
null);
tdsWriter.writeNonUnicodeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH, true);
} else {
SQLCollation destCollation = destColumnMetadata.get(destColOrdinal).collation;
if (null != destCollation) {
tdsWriter.writeNonUnicodeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH,
false, destCollation.getCharset());
false);
} else {
tdsWriter.writeNonUnicodeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH,
false, null);
false);
}
}
}
Expand Down Expand Up @@ -3367,7 +3367,8 @@ private byte[] getEncryptedTemporalBytes(TDSWriter tdsWriter, JDBCType srcTempor
if (subSecondNanos < 0)
subSecondNanos += Nanos.PER_SECOND;
}
return tdsWriter.writeEncryptedScaledTemporal(calendar, subSecondNanos, scale, SSType.TIME, (short) 0, null);
return tdsWriter.writeEncryptedScaledTemporal(calendar, subSecondNanos, scale, SSType.TIME, (short) 0,
null);

case TIMESTAMP:
calendar = new GregorianCalendar(java.util.TimeZone.getDefault(), java.util.Locale.US);
Expand Down
88 changes: 80 additions & 8 deletions src/test/java/com/microsoft/sqlserver/jdbc/bulkCopy/BulkCopyColumnMappingTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import com.microsoft.sqlserver.jdbc.ComparisonUtil;
import com.microsoft.sqlserver.jdbc.RandomUtil;
import com.microsoft.sqlserver.jdbc.SQLServerBulkCopy;
import com.microsoft.sqlserver.jdbc.SQLServerBulkCopyOptions;
import com.microsoft.sqlserver.jdbc.TestResource;
import com.microsoft.sqlserver.jdbc.TestUtils;
import com.microsoft.sqlserver.testframework.AbstractSQLGenerator;
Expand All @@ -44,6 +43,7 @@
@RunWith(JUnitPlatform.class)
@DisplayName("BulkCopy Column Mapping Test")
public class BulkCopyColumnMappingTest extends BulkCopyTestSetUp {
private String prcdb = RandomUtil.getIdentifier("BulkCopy_PRC_DB");

@Test
@DisplayName("BulkCopy:test no explicit column mapping")
Expand All @@ -62,6 +62,8 @@ public void testNoExplicitCM() throws SQLException {
bulkWrapper.setUsingXAConnection((0 == Constants.RANDOM.nextInt(2)) ? true : false, dsXA);
bulkWrapper.setUsingPooledConnection((0 == Constants.RANDOM.nextInt(2)) ? true : false, dsPool);
BulkCopyTestUtil.performBulkCopy(bulkWrapper, sourceTable, destTable);
} catch (Exception e) {
fail(e.getMessage());
} finally {
TestUtils.dropTableIfExists(destTable.getEscapedTableName(), (Statement) stmt.product());
}
Expand Down Expand Up @@ -103,6 +105,8 @@ public void testExplicitCM() throws SQLException {
}
}
BulkCopyTestUtil.performBulkCopy(bulkWrapper, sourceTable, destTable);
} catch (Exception e) {
fail(e.getMessage());
} finally {
TestUtils.dropTableIfExists(destTable.getEscapedTableName(), (Statement) stmt.product());
}
Expand Down Expand Up @@ -148,6 +152,8 @@ public void testUnicodeCM() throws SQLException {
}
}
BulkCopyTestUtil.performBulkCopy(bulkWrapper, sourceTableUnicode, destTableUnicode);
} catch (Exception e) {
fail(e.getMessage());
} finally {
TestUtils.dropTableIfExists(sourceTableUnicode.getEscapedTableName(), (Statement) stmt.product());
TestUtils.dropTableIfExists(destTableUnicode.getEscapedTableName(), (Statement) stmt.product());
Expand Down Expand Up @@ -213,6 +219,8 @@ public void testRepetitiveCM() throws SQLException {

fail(form.format(msgArgs) + "\n" + destTable.getTableName() + "\n" + e.getMessage());
}
} catch (Exception e) {
fail(e.getMessage());
} finally {
TestUtils.dropTableIfExists(sourceTable1.getEscapedTableName(), (Statement) stmt.product());
TestUtils.dropTableIfExists(destTable.getEscapedTableName(), (Statement) stmt.product());
Expand Down Expand Up @@ -254,6 +262,8 @@ public void testImplicitMismatchCM() throws SQLException {
}
}
BulkCopyTestUtil.performBulkCopy(bulkWrapper, sourceTable, destTable, true, true);
} catch (Exception e) {
fail(e.getMessage());
} finally {
TestUtils.dropTableIfExists(destTable.getEscapedTableName(), (Statement) stmt.product());
}
Expand Down Expand Up @@ -361,6 +371,8 @@ public void testInvalidCM() throws SQLException {
bulkWrapper.setUsingPooledConnection((0 == Constants.RANDOM.nextInt(2)) ? true : false, dsPool);
bulkWrapper.setColumnMapping(Integer.MIN_VALUE, Integer.MAX_VALUE);
BulkCopyTestUtil.performBulkCopy(bulkWrapper, sourceTable, destTable, true, true);
} catch (Exception e) {
fail(e.getMessage());
} finally {
TestUtils.dropTableIfExists(destTable.getEscapedTableName(), (Statement) stmt.product());
}
Expand All @@ -374,11 +386,33 @@ public void testUnicodeCharToNchar() throws SQLException, ClassNotFoundException
// Windows only as this test has problems getting dependent H2 package from Maven in Unix
org.junit.Assume.assumeTrue(isWindows);

validateMapping("CHAR(5)", "NCHAR(5)", "фщыab");
validateMapping("CHAR(5)", "NVARCHAR(5)", "фщыab");
validateMapping("VARCHAR(5)", "NCHAR(5)", "фщыab");
validateMapping("VARCHAR(5)", "NVARCHAR(5)", "фщыab");
validateMapping("VARCHAR(5)", "NVARCHAR(max)", "фщыab");
validateNMapping("CHAR(5)", "NCHAR(5)", "фщыab");
validateNMapping("CHAR(5)", "NVARCHAR(5)", "фщыab");
validateNMapping("VARCHAR(5)", "NCHAR(5)", "фщыab");
validateNMapping("VARCHAR(5)", "NVARCHAR(5)", "фщыab");
validateNMapping("VARCHAR(5)", "NVARCHAR(max)", "фщыab");
}

@Tag(Constants.xAzureSQLDW)
@Tag(Constants.xAzureSQLDB)
@Test
@DisplayName("BulkCopy:test unicode char/varchar to char/varchar")
public void testUnicodeCharToChar() throws SQLException, ClassNotFoundException {

try (Connection con = DriverManager.getConnection(connectionString); Statement stmt = con.createStatement()) {
stmt.executeUpdate("CREATE DATABASE [" + prcdb + "]");
stmt.executeUpdate("ALTER DATABASE " + "[" + prcdb + "]" + " COLLATE Chinese_PRC_CI_AS;");

validateMapping("CHAR(10)", "CHAR(10)", "测试设计者");
validateMapping("CHAR(10)", "VARCHAR(10)", "测试设计者");
validateMapping("VARCHAR(10)", "CHAR(10)", "测试设计者");
validateMapping("VARCHAR(10)", "VARCHAR(10)", "测试设计者");
validateMapping("VARCHAR(10)", "VARCHAR(max)", "测试设计者");
} catch (Exception e) {
fail(e.getMessage());
} finally {
TestUtils.dropDatabaseIfExists(prcdb, connectionString);
}
}

@Tag(Constants.xAzureSQLDW)
Expand Down Expand Up @@ -466,7 +500,8 @@ private void validateValuesRepetitiveCM(DBConnection con, DBTable sourceTable,
}
}

private void validateMapping(String sourceType, String destType,
// validate mapping from Char/Varchar to NChar/Varchar types
private void validateNMapping(String sourceType, String destType,
String data) throws SQLException, ClassNotFoundException {
Class.forName("org.h2.Driver");
Random rand = new Random();
Expand All @@ -483,14 +518,51 @@ private void validateMapping(String sourceType, String destType,
sourceStmt.executeUpdate("CREATE TABLE " + sourceTable + " (col " + sourceType + ");");
sourceStmt.executeUpdate("INSERT INTO " + sourceTable + " VALUES('" + data + "');");

destStmt.executeUpdate("CREATE TABLE " + destTable + " (col NCHAR(5));");
destStmt.executeUpdate("CREATE TABLE " + destTable + " (col " + destType + ");");

ResultSet sourceRs = sourceStmt.executeQuery("SELECT * FROM " + sourceTable);
bulkCopy.writeToServer(sourceRs);

ResultSet destRs = destStmt.executeQuery("SELECT * FROM " + destTable);
destRs.next();
String receivedUnicodeData = destRs.getString(1);
assertEquals(data, receivedUnicodeData);
} catch (Exception e) {
fail(e.getMessage());
} finally {
sourceStmt.executeUpdate("DROP TABLE " + sourceTable);
TestUtils.dropTableIfExists(destTable, destStmt);
}
}
}

private void validateMapping(String sourceType, String destType,
String data) throws SQLException, ClassNotFoundException {
Random rand = new Random();
String sourceTable = "sourceTable" + rand.nextInt(Integer.MAX_VALUE);
String destTable = TestUtils
.escapeSingleQuotes(AbstractSQLGenerator.escapeIdentifier(RandomUtil.getIdentifier("destTable")));
try (Connection sourceCon = DriverManager.getConnection(connectionString + ";database=" + prcdb);
Connection destCon = DriverManager.getConnection(connectionString + ";database=" + prcdb);
Statement sourceStmt = sourceCon.createStatement(); Statement destStmt = destCon.createStatement();
SQLServerBulkCopy bulkCopy = new SQLServerBulkCopy(destCon)) {
try {
bulkCopy.setDestinationTableName(destTable);

sourceStmt.executeUpdate("CREATE TABLE " + sourceTable + " (col " + sourceType + ");");
sourceStmt.executeUpdate("INSERT INTO " + sourceTable + " VALUES('" + data + "');");

destStmt.executeUpdate("CREATE TABLE " + destTable + " (col " + destType + ");");

ResultSet sourceRs = sourceStmt.executeQuery("SELECT * FROM " + sourceTable);
bulkCopy.writeToServer(sourceRs);

ResultSet destRs = destStmt.executeQuery("SELECT * FROM " + destTable);
destRs.next();
String receivedUnicodeData = destRs.getString(1);
assertEquals(data, receivedUnicodeData);
} catch (Exception e) {
fail(e.getMessage());
} finally {
sourceStmt.executeUpdate("DROP TABLE " + sourceTable);
TestUtils.dropTableIfExists(destTable, destStmt);
Expand Down

0 comments on commit 4a46e6c

Please sign in to comment.