Skip to content

Commit

Permalink
0006170: DB Compare incorrectly comparing values
Browse files Browse the repository at this point in the history
  • Loading branch information
jakobvanmeter committed Feb 16, 2024
1 parent a763d34 commit 3607838
Show file tree
Hide file tree
Showing 3 changed files with 89 additions and 31 deletions.
82 changes: 66 additions & 16 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompare.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStream;
import java.sql.Types;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
Expand All @@ -43,8 +44,11 @@
import org.jumpmind.db.sql.Row;
import org.jumpmind.symmetric.ISymmetricEngine;
import org.jumpmind.symmetric.SymmetricException;
import org.jumpmind.symmetric.common.ParameterConstants;
import org.jumpmind.symmetric.common.TableConstants;
import org.jumpmind.symmetric.db.ISymmetricDialect;
import org.jumpmind.symmetric.io.DbCompareReport.TableReport;
import org.jumpmind.symmetric.load.IReloadVariableFilter;
import org.jumpmind.symmetric.model.TriggerHistory;
import org.jumpmind.symmetric.model.TriggerRouter;
import org.jumpmind.symmetric.service.impl.TransformService.TransformTableNodeGroupLink;
Expand Down Expand Up @@ -137,7 +141,6 @@ protected OutputStream getSqlDiffOutputStream(DbCompareTables tables) {
protected TableReport compareTables(DbCompareTables tables, OutputStream sqlDiffOutput) {
String sourceSelect = getSourceComparisonSQL(tables, sourceEngine.getTargetDialect().getTargetPlatform());
String targetSelect = getTargetComparisonSQL(tables, targetEngine.getTargetDialect().getTargetPlatform());
// String targetSelect = getTargetComparisonSQL(tables, targetEngine.getDatabasePlatform());
CountingSqlReadCursor sourceCursor = new CountingSqlReadCursor(sourceEngine.getTargetDialect().getTargetPlatform().getSqlTemplateDirty().queryForCursor(
sourceSelect,
defaultRowMapper));
Expand Down Expand Up @@ -272,19 +275,49 @@ protected String getTargetComparisonSQL(DbCompareTables tables, IDatabasePlatfor
return sql;
}

protected String getComparisonSQL(Table table, Column[] sortByColumns, IDatabasePlatform platform,
String whereClause, boolean isSource) {
DmlStatement statement = platform.createDmlStatement(DmlType.SELECT,
table.getCatalog(), table.getSchema(), table.getName(),
null, table.getColumns(),
null, null);
StringBuilder sql = new StringBuilder(statement.getSql());
sql.setLength(sql.length() - "where ".length()); // remove the trailing where so we can insert a table alias.
sql.append(" t where "); // main table alias.
sql.append(whereClause).append(" ");
sql.append(buildOrderBy(table, sortByColumns, platform, isSource));
return sql.toString();
}
protected String getComparisonSQL(Table table, Column[] sortByColumns, IDatabasePlatform platform,
String whereClause, boolean isSource) {

DmlStatement statement = platform.createDmlStatement(DmlType.SELECT, table.getCatalog(), table.getSchema(),
table.getName(), null, table.getColumns(), null, null);
StringBuilder sql = new StringBuilder(statement.getSql());
String sybaseUnitypeConversions = statement.getSql();
boolean isUsingUnitypes = false;
if (platform.getSqlTemplate().toString().contains("ase")) {
if (isSource) {
ISymmetricDialect symmetricDialect = sourceEngine.getSymmetricDialect();
isUsingUnitypes = symmetricDialect.getParameterService()
.is(ParameterConstants.DBDIALECT_SYBASE_ASE_CONVERT_UNITYPES_FOR_SYNC);
} else {
ISymmetricDialect symmetricDialect = targetEngine.getSymmetricDialect();
isUsingUnitypes = symmetricDialect.getParameterService()
.is(ParameterConstants.DBDIALECT_SYBASE_ASE_CONVERT_UNITYPES_FOR_SYNC);
}
if (isUsingUnitypes) {
for(Column column : statement.getColumns()) {
if(column.getJdbcTypeName().equalsIgnoreCase("unichar") ||
column.getJdbcTypeName().equalsIgnoreCase("univarchar") ||
column.getJdbcTypeName().equalsIgnoreCase("unitext")) {
sybaseUnitypeConversions = sybaseUnitypeConversions.replace(column.getName(),"case when " + column.getName() + " is null then null else '\"' +\n"
+ " bintostr(convert(varbinary(16384),"+column.getName()+")) + '\"' end as " +column.getName() );

}
}

}
}
StringBuilder finalSql = null;
if(isUsingUnitypes) {
finalSql = new StringBuilder(sybaseUnitypeConversions.toString());
} else {
finalSql = new StringBuilder(sql.toString());
}
finalSql.setLength(finalSql.length() - "where ".length()); // remove the trailing where so we can insert a table alias.
finalSql.append(" t where "); // main table alias.
finalSql.append(whereClause).append(" ");
finalSql.append(buildOrderBy(table, sortByColumns, platform, isSource));
return finalSql.toString();
}

protected String buildOrderBy(Table table, Column[] sortByColumns, IDatabasePlatform platform, boolean isSource) {
DatabaseInfo databaseInfo = platform.getDatabaseInfo();
Expand Down Expand Up @@ -398,6 +431,14 @@ protected List<DbCompareTables> loadTables(List<String> tableNames, List<String>
}
}
Table sourceTableCopy = sourceTable.copy();
for (Column column : sourceTableCopy.getColumns()) {
if (column.getJdbcTypeName().equalsIgnoreCase("univarchar") ||
column.getJdbcTypeName().equalsIgnoreCase("unichar") ||
column.getJdbcTypeName().equalsIgnoreCase("unitext")) {
column.setMappedType("VARCHAR");
column.setMappedTypeCode(Types.VARCHAR);
}
}
DbCompareTables tables = new DbCompareTables(sourceTableCopy, null);
String targetTableName = tableName;
if (!CollectionUtils.isEmpty(targetTableNames)) {
Expand Down Expand Up @@ -489,8 +530,17 @@ protected Table loadTargetTable(DbCompareTables tables, String targetTableName)
}
}
}
tables.setTargetTable(targetTable);
return targetTable;
Table targetTableCopy = targetTable.copy();
// for (Column column : targetTableCopy.getColumns()) {
// if (column.getJdbcTypeName().equalsIgnoreCase("univarchar") ||
// column.getJdbcTypeName().equalsIgnoreCase("unichar") ||
// column.getJdbcTypeName().equalsIgnoreCase("unitext")) {
// column.setMappedType("VARCHAR");
// column.setMappedTypeCode(Types.VARCHAR);
// }
// }
tables.setTargetTable(targetTableCopy);
return targetTableCopy;
}

protected TriggerRouter getTriggerRouterFor(Table sourceTable) {
Expand Down
15 changes: 0 additions & 15 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompareDiffWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,6 @@ public void writeDelete(DbCompareRow targetCompareRow) {
null, null);
Row row = new Row(targetCompareRow.getTable().getPrimaryKeyColumnCount());
for (int i = 0; i < targetCompareRow.getTable().getPrimaryKeyColumnCount(); i++) {
if (table.getColumn(i).getJdbcTypeName().equalsIgnoreCase("univarchar") ||
table.getColumn(i).getJdbcTypeName().equalsIgnoreCase("unichar") ||
table.getColumn(i).getJdbcTypeName().equalsIgnoreCase("unitext")) {
table.getColumn(i).setMappedType("VARCHAR");
}
row.put(table.getColumn(i).getName(),
targetCompareRow.getRowValues().get(targetCompareRow.getTable().getColumn(i).getName()));
}
Expand Down Expand Up @@ -113,11 +108,6 @@ public void writeInsert(DbCompareRow sourceCompareRow) {
if (targetColumn == null) {
continue;
}
if (targetColumn.getJdbcTypeName().equalsIgnoreCase("univarchar") ||
targetColumn.getJdbcTypeName().equalsIgnoreCase("unichar") ||
targetColumn.getJdbcTypeName().equalsIgnoreCase("unitext")) {
targetColumn.setMappedType("VARCHAR");
}
row.put(targetColumn.getName(), sourceCompareRow.getRowValues().get(sourceColumn.getName()));
}
String sql = statement.buildDynamicSql(BinaryEncoding.HEX, row, false, false);
Expand Down Expand Up @@ -145,11 +135,6 @@ public void writeUpdate(DbCompareRow targetCompareRow, Map<Column, String> delta
null, null);
Row row = new Row(changedColumns.length + table.getPrimaryKeyColumnCount());
for (Column changedColumn : deltas.keySet()) {
if (changedColumn.getJdbcTypeName().equalsIgnoreCase("univarchar") ||
changedColumn.getJdbcTypeName().equalsIgnoreCase("unichar") ||
changedColumn.getJdbcTypeName().equalsIgnoreCase("unitext")) {
changedColumn.setMappedType("VARCHAR");
}
String value = deltas.get(changedColumn);
row.put(changedColumn.getName(), value);
}
Expand Down
23 changes: 23 additions & 0 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbValueComparator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,20 @@
*/
package org.jumpmind.symmetric.io;

import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.nio.charset.Charset;
import java.sql.Types;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;

import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.commons.lang3.time.DateUtils;
Expand Down Expand Up @@ -64,6 +69,11 @@ public int compareValues(Column sourceColumn, Column targetColumn, String source
return 0;
}
if (sourceColumn.isOfTextType()) {
if(targetColumn.getJdbcTypeName().equalsIgnoreCase("univarchar") ||
targetColumn.getJdbcTypeName().equalsIgnoreCase("unichar") ||
targetColumn.getJdbcTypeName().equalsIgnoreCase("unitext")) {
targetValue = convertString(targetValue);
}
return compareText(sourceColumn, targetColumn, sourceValue, targetValue);
} else if (sourceColumn.isOfNumericType()) {
return compareNumeric(sourceColumn, targetColumn, sourceValue, targetValue);
Expand All @@ -73,6 +83,19 @@ public int compareValues(Column sourceColumn, Column targetColumn, String source
return compareDefault(sourceColumn, targetColumn, sourceValue, targetValue);
}
}

protected String convertString(String string) {
String utf8String = null;
try {
string = new String(Hex.decodeHex(string));
string = string.substring(1,string.length()-1);
string = "fffe" + string;
utf8String = new String(Hex.decodeHex(string), "UTF-16");
} catch (DecoderException | UnsupportedEncodingException e) {
e.printStackTrace();
}
return utf8String;
}

public int compareText(Column sourceColumn, Column targetColumn, String source, String target) {
if (stringNullEqualsEmptyString) {
Expand Down

0 comments on commit 3607838

Please sign in to comment.