Skip to content

Commit

Permalink
Performance optimizations when bulk loading large amounts of timestam…
Browse files Browse the repository at this point in the history 
…ps (#2194)
  • Loading branch information
@hannes92
hannes92 committed Nov 2, 2023
1 parent 974e4a1 commit a762d7c
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 22 deletions.
26 changes: 11 additions & 15 deletions src/main/java/com/microsoft/sqlserver/jdbc/IOBuffer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import java.sql.Timestamp;
import java.text.MessageFormat;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.OffsetTime;
import java.util.ArrayList;
Expand Down Expand Up @@ -3755,29 +3756,24 @@ void writeSmalldatetime(String value) throws SQLServerException {
writeShort((short) minutesSinceMidnight);
}

void writeDatetime(String value) throws SQLServerException {
GregorianCalendar calendar = initializeCalender(TimeZone.getDefault());
long utcMillis; // Value to which the calendar is to be set (in milliseconds 1/1/1970 00:00:00 GMT)
void writeDatetime(java.sql.Timestamp dateValue) throws SQLServerException {
LocalDateTime ldt = dateValue.toLocalDateTime();
int subSecondNanos;
java.sql.Timestamp timestampValue = java.sql.Timestamp.valueOf(value);
utcMillis = timestampValue.getTime();
subSecondNanos = timestampValue.getNanos();
subSecondNanos = ldt.getNano();

// Load the calendar with the desired value
calendar.setTimeInMillis(utcMillis);

// Number of days there have been since the SQL Base Date.
// These are based on SQL Server algorithms
int daysSinceSQLBaseDate = DDC.daysSinceBaseDate(calendar.get(Calendar.YEAR),
calendar.get(Calendar.DAY_OF_YEAR), TDS.BASE_YEAR_1900);
int daysSinceSQLBaseDate = DDC.daysSinceBaseDate(ldt.getYear(),
ldt.getDayOfYear(), TDS.BASE_YEAR_1900);

// Number of milliseconds since midnight of the current day.
int millisSinceMidnight = (subSecondNanos + Nanos.PER_MILLISECOND / 2) / Nanos.PER_MILLISECOND + // Millis into
// the current
// second
1000 * calendar.get(Calendar.SECOND) + // Seconds into the current minute
60 * 1000 * calendar.get(Calendar.MINUTE) + // Minutes into the current hour
60 * 60 * 1000 * calendar.get(Calendar.HOUR_OF_DAY); // Hours into the current day
// the current
// second
1000 * ldt.getSecond() + // Seconds into the current minute
60 * 1000 * ldt.getMinute() + // Minutes into the current hour
60 * 60 * 1000 * ldt.getHour(); // Hours into the current day

// The last millisecond of the current day is always rounded to the first millisecond
// of the next day because DATETIME is only accurate to 1/300th of a second.
Expand Down
19 changes: 12 additions & 7 deletions src/main/java/com/microsoft/sqlserver/jdbc/SQLServerBulkCopy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2463,7 +2463,12 @@ else if (null != sourceCryptoMeta) {
case DATETIME:
if (bulkNullable)
tdsWriter.writeByte((byte) 0x08);
tdsWriter.writeDatetime(colValue.toString());

if (colValue instanceof java.sql.Timestamp) {
tdsWriter.writeDatetime((java.sql.Timestamp) colValue);
} else {
tdsWriter.writeDatetime(java.sql.Timestamp.valueOf(colValue.toString()));
}
break;
default: // DATETIME2
if (2 >= bulkScale)
Expand Down Expand Up @@ -2697,15 +2702,15 @@ private void writeSqlVariant(TDSWriter tdsWriter, Object colValue, ResultSet sou
tdsWriter.writeTime((java.sql.Timestamp) colValue, timeBulkScale);
break;

case DATETIME8:
writeBulkCopySqlVariantHeader(10, TDSType.DATETIME8.byteValue(), (byte) 0, tdsWriter);
tdsWriter.writeDatetime(colValue.toString());
break;

case DATETIME4:
// when the type is ambiguous, we write to bigger type
case DATETIME8:
writeBulkCopySqlVariantHeader(10, TDSType.DATETIME8.byteValue(), (byte) 0, tdsWriter);
tdsWriter.writeDatetime(colValue.toString());
if (colValue instanceof java.sql.Timestamp) {
tdsWriter.writeDatetime((java.sql.Timestamp) colValue);
} else {
tdsWriter.writeDatetime(java.sql.Timestamp.valueOf(colValue.toString()));
}
break;

case DATETIME2N:
Expand Down
85 changes: 85 additions & 0 deletions src/test/java/com/microsoft/sqlserver/jdbc/bulkCopy/BulkCopyAllTypesTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.sql.Types;

import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
Expand All @@ -16,15 +18,33 @@
import org.junit.runner.RunWith;

import com.microsoft.sqlserver.jdbc.ComparisonUtil;
import com.microsoft.sqlserver.jdbc.RandomData;
import com.microsoft.sqlserver.jdbc.RandomUtil;
import com.microsoft.sqlserver.jdbc.SQLServerBulkCopy;
import com.microsoft.sqlserver.jdbc.SQLServerBulkCopyOptions;
import com.microsoft.sqlserver.jdbc.TestUtils;
import com.microsoft.sqlserver.testframework.AbstractSQLGenerator;
import com.microsoft.sqlserver.testframework.AbstractTest;
import com.microsoft.sqlserver.testframework.Constants;
import com.microsoft.sqlserver.testframework.DBConnection;
import com.microsoft.sqlserver.testframework.DBStatement;
import com.microsoft.sqlserver.testframework.DBTable;
import com.microsoft.sqlserver.testframework.PrepUtil;

import javax.sql.RowSetMetaData;
import javax.sql.rowset.CachedRowSet;
import javax.sql.rowset.RowSetFactory;
import javax.sql.rowset.RowSetMetaDataImpl;
import javax.sql.rowset.RowSetProvider;

import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;


@RunWith(JUnitPlatform.class)
public class BulkCopyAllTypesTest extends AbstractTest {
Expand Down Expand Up @@ -98,6 +118,71 @@ private void terminateVariation() throws SQLException {
try (Statement stmt = connection.createStatement()) {
TestUtils.dropTableIfExists(tableSrc.getEscapedTableName(), stmt);
TestUtils.dropTableIfExists(tableDest.getEscapedTableName(), stmt);
TestUtils.dropTableIfExists(dateTimeTestTable, stmt);
}
}

private static final int DATETIME_COL_COUNT = 2;
private static final int DATETIME_ROW_COUNT = 1;
private static final String dateTimeTestTable =
AbstractSQLGenerator.escapeIdentifier(RandomUtil.getIdentifier("bulkCopyTimestampTest"));

@Test
public void testBulkCopyTimestamp() throws SQLException {
List<Timestamp> timeStamps = new ArrayList<>();
try (Connection con = getConnection(); Statement stmt = connection.createStatement()) {
String colSpec = IntStream.range(1, DATETIME_COL_COUNT + 1).mapToObj(x -> String.format("c%d datetime", x)).collect(
Collectors.joining(","));
String sql1 = String.format("create table %s (%s)", dateTimeTestTable, colSpec);
stmt.execute(sql1);

RowSetFactory rsf = RowSetProvider.newFactory();
CachedRowSet crs = rsf.createCachedRowSet();
RowSetMetaData rsmd = new RowSetMetaDataImpl();
rsmd.setColumnCount(DATETIME_COL_COUNT);

for (int i = 1; i <= DATETIME_COL_COUNT; i++) {
rsmd.setColumnName(i, String.format("c%d", i));
rsmd.setColumnType(i, Types.TIMESTAMP);
}
crs.setMetaData(rsmd);

for (int i = 0; i < DATETIME_COL_COUNT; i++) {
timeStamps.add(RandomData.generateDatetime(false));
}

for (int ri = 0; ri < DATETIME_ROW_COUNT; ri++) {
crs.moveToInsertRow();

for (int i = 1; i <= DATETIME_COL_COUNT; i++) {
crs.updateTimestamp(i, timeStamps.get(i - 1));
}
crs.insertRow();
}
crs.moveToCurrentRow();

try (SQLServerBulkCopy bcOperation = new SQLServerBulkCopy(con)) {
SQLServerBulkCopyOptions bcOptions = new SQLServerBulkCopyOptions();
bcOptions.setBatchSize(5000);
bcOperation.setDestinationTableName(dateTimeTestTable);
bcOperation.setBulkCopyOptions(bcOptions);
bcOperation.writeToServer(crs);
}

try (ResultSet rs = stmt.executeQuery("select * from " + dateTimeTestTable)) {
assertTrue(rs.next());

for (int i = 1; i <= DATETIME_COL_COUNT; i++) {
long expectedTimestamp = getTime(timeStamps.get(i - 1));
long actualTimestamp = getTime(rs.getTimestamp(i));

assertEquals(expectedTimestamp, actualTimestamp);
}
}
}
}

private static long getTime(Timestamp time) {
return (3 * time.getTime() + 5) / 10;
}
}

0 comments on commit a762d7c

Please sign in to comment.