Skip to content

Commit

Permalink
0001959: Add feature where character data can be modified by an prope…
Browse files Browse the repository at this point in the history 
…rty expression in order to support "weird" character issues
  • Loading branch information
@chenson42
chenson42 committed Sep 11, 2014
1 parent 9a80538 commit 39b95bb
Show file tree
Hide file tree
Showing 20 changed files with 96 additions and 46 deletions.
2 changes: 1 addition & 1 deletion symmetric-client/src/test/java/org/jumpmind/symmetric/DbExportImportTest.java
View file
Expand Up @@ -224,7 +224,7 @@ protected boolean createAndFillTimestampWithTimeZoneTable() {
TEST_TS_W_TZ);
template.update(createSql);
DmlStatement statement = platform.createDmlStatement(DmlType.INSERT,
platform.getTableFromCache(TEST_TS_W_TZ, true));
platform.getTableFromCache(TEST_TS_W_TZ, true), null);
template.update(statement.getSql(), statement.getValueArray(new Object[] { 1,
"1973-06-08 07:00:00.000 -04:00" }, new Object[] { 1 }));
return true;
Expand Down
1 change: 1 addition & 0 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java
View file
Expand Up @@ -152,6 +152,7 @@ private ParameterConstants() {
public final static String DATA_LOADER_TIME_BETWEEN_ACK_RETRIES = "time.between.ack.retries.ms";
public final static String DATA_LOADER_MAX_ROWS_BEFORE_COMMIT = "dataloader.max.rows.before.commit";
public final static String DATA_LOADER_CREATE_TABLE_ALTER_TO_MATCH_DB_CASE = "dataloader.create.table.alter.to.match.db.case";
public final static String DATA_LOADER_TEXT_COLUMN_EXPRESSION = "dataloader.text.column.expression";
public final static String DATA_LOADER_SLEEP_TIME_AFTER_EARLY_COMMIT = "dataloader.sleep.time.after.early.commit";
public final static String DATA_LOADER_TREAT_DATETIME_AS_VARCHAR = "db.treat.date.time.as.varchar.enabled";

Expand Down
2 changes: 2 additions & 0 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/load/DefaultDataLoaderFactory.java
View file
Expand Up @@ -117,6 +117,8 @@ protected DatabaseWriterSettings buildDatabaseWriterSettings(
settings.setSaveCurrentValueOnError(parameterService.is(
ParameterConstants.DATA_LOADER_ERROR_RECORD_CUR_VAL, false));
settings.setFitToColumn(parameterService.is(ParameterConstants.DATA_LOADER_FIT_TO_COLUMN, false));
settings.setTextColumnExpression(parameterService.getString(
ParameterConstants.DATA_LOADER_TEXT_COLUMN_EXPRESSION));

Map<String, Conflict> byChannel = new HashMap<String, Conflict>();
Map<String, Conflict> byTable = new HashMap<String, Conflict>();
Expand Down
2 changes: 1 addition & 1 deletion symmetric-core/src/main/java/org/jumpmind/symmetric/route/AuditTableDataRouter.java
View file
Expand Up @@ -102,7 +102,7 @@ public Set<String> routeToNodes(SimpleRouterContext context, DataMetaData dataMe
values.put(auditTable.getColumnWithName(COLUMN_AUDIT_ID).getName(), sequence);
values.put(auditTable.getColumnWithName(COLUMN_AUDIT_TIME).getName(), new Date());
values.put(auditTable.getColumnWithName(COLUMN_AUDIT_EVENT).getName(), eventType.getCode());
DmlStatement statement = platform.createDmlStatement(DmlType.INSERT, auditTable);
DmlStatement statement = platform.createDmlStatement(DmlType.INSERT, auditTable, null);
int[] types = statement.getTypes();
Object[] args = statement.getValueArray(values);
String sql = statement.getSql();
Expand Down
12 changes: 12 additions & 0 deletions symmetric-core/src/main/resources/symmetric-default.properties
View file
Expand Up @@ -982,6 +982,18 @@ dataloader.create.table.alter.to.match.db.case=true
# Type: boolean
dataloader.error.save.curval=false

# Provide a SQL expression that will be used by the data loader in DML statements
# for all text based column values (like varchar, char, nvarchar,
# clob and nchar columns). The expression can be used to make scenario based casts. For example,
# if the data in the database should be converted to a different character set that the default character set
# on Oracle, then a helpful expression might be something like this:
# convert($(columnName), 'AR8MSWIN1256', 'AR8ISO8859P6')
#
# DatabaseOverridable: true
# Tags: load
# Type: textbox
dataloader.text.column.expression=

# The number of milliseconds parameters will be cached by the ParameterService before they are reread from the
# file system and database.
#
Expand Down
8 changes: 4 additions & 4 deletions symmetric-db/src/main/java/org/jumpmind/db/platform/AbstractDatabasePlatform.java
View file
Expand Up @@ -107,15 +107,15 @@ public DatabaseInfo getDatabaseInfo() {

abstract public ISqlTemplate getSqlTemplate();

public DmlStatement createDmlStatement(DmlType dmlType, Table table) {
public DmlStatement createDmlStatement(DmlType dmlType, Table table, String textColumnExpression) {
return createDmlStatement(dmlType, table.getCatalog(), table.getSchema(), table.getName(),
table.getPrimaryKeyColumns(), table.getColumns(), null);
table.getPrimaryKeyColumns(), table.getColumns(), null, textColumnExpression);
}

public DmlStatement createDmlStatement(DmlType dmlType, String catalogName, String schemaName,
String tableName, Column[] keys, Column[] columns, boolean[] nullKeyValues) {
String tableName, Column[] keys, Column[] columns, boolean[] nullKeyValues, String textColumnExpression) {
return DmlStatementFactory.createDmlStatement(getName(), dmlType, catalogName, schemaName,
tableName, keys, columns, nullKeyValues, getDdlBuilder());
tableName, keys, columns, nullKeyValues, getDdlBuilder(), textColumnExpression);
}

public IDdlReader getDdlReader() {
Expand Down
18 changes: 9 additions & 9 deletions symmetric-db/src/main/java/org/jumpmind/db/platform/DmlStatementFactory.java
View file
Expand Up @@ -53,41 +53,41 @@ public static DmlStatement createDmlStatement(String databaseName, DmlType dmlTy
} else {
ddlBuilder.setDelimitedIdentifierModeOn(useQuotedIdentifiers);
return createDmlStatement(databaseName, dmlType, catalogName, schemaName, tableName, keys,
columns, nullKeyValues, ddlBuilder);
columns, nullKeyValues, ddlBuilder, null);
}
}

public static DmlStatement createDmlStatement(String databaseName, DmlType dmlType,
String catalogName, String schemaName, String tableName, Column[] keys,
Column[] columns, boolean[] nullKeyValues, IDdlBuilder ddlBuilder) {
Column[] columns, boolean[] nullKeyValues, IDdlBuilder ddlBuilder, String textColumnExpression) {
if (DatabaseNamesConstants.ORACLE.equals(databaseName)) {
return new OracleDmlStatement(dmlType, catalogName, schemaName, tableName, keys,
columns, nullKeyValues, ddlBuilder.getDatabaseInfo(),
ddlBuilder.isDelimitedIdentifierModeOn());
ddlBuilder.isDelimitedIdentifierModeOn(), textColumnExpression);
} else if (DatabaseNamesConstants.POSTGRESQL.equals(databaseName)) {
return new PostgreSqlDmlStatement(dmlType, catalogName, schemaName, tableName, keys,
columns, nullKeyValues, ddlBuilder.getDatabaseInfo(),
ddlBuilder.isDelimitedIdentifierModeOn());
ddlBuilder.isDelimitedIdentifierModeOn(), textColumnExpression);
} else if (DatabaseNamesConstants.REDSHIFT.equals(databaseName)) {
return new RedshiftDmlStatement(dmlType, catalogName, schemaName, tableName, keys,
columns, nullKeyValues, ddlBuilder.getDatabaseInfo(),
ddlBuilder.isDelimitedIdentifierModeOn());
ddlBuilder.isDelimitedIdentifierModeOn(), textColumnExpression);
} else if (DatabaseNamesConstants.MYSQL.equals(databaseName)) {
return new MySqlDmlStatement(dmlType, catalogName, schemaName, tableName, keys,
columns, nullKeyValues, ddlBuilder.getDatabaseInfo(),
ddlBuilder.isDelimitedIdentifierModeOn());
ddlBuilder.isDelimitedIdentifierModeOn(), textColumnExpression);
} else if (DatabaseNamesConstants.SQLITE.equals(databaseName)) {
return new SqliteDmlStatement(dmlType, catalogName, schemaName, tableName, keys,
columns, nullKeyValues, ddlBuilder.getDatabaseInfo(),
ddlBuilder.isDelimitedIdentifierModeOn());
ddlBuilder.isDelimitedIdentifierModeOn(), textColumnExpression);
} else if (DatabaseNamesConstants.SQLANYWHERE.equals(databaseName)) {
return new SqlAnywhereDmlStatement(dmlType, catalogName, schemaName, tableName, keys, columns,
nullKeyValues, ddlBuilder.getDatabaseInfo(),
ddlBuilder.isDelimitedIdentifierModeOn());
ddlBuilder.isDelimitedIdentifierModeOn(), textColumnExpression);
} else {
return new DmlStatement(dmlType, catalogName, schemaName, tableName, keys, columns,
nullKeyValues, ddlBuilder.getDatabaseInfo(),
ddlBuilder.isDelimitedIdentifierModeOn());
ddlBuilder.isDelimitedIdentifierModeOn(), textColumnExpression);
}

}
Expand Down
4 changes: 2 additions & 2 deletions symmetric-db/src/main/java/org/jumpmind/db/platform/IDatabasePlatform.java
View file
Expand Up @@ -112,10 +112,10 @@ public void createTables(boolean dropTablesFirst,

public void dropTables(boolean continueOnError, Table...tables);

public DmlStatement createDmlStatement(DmlType dmlType, Table table);
public DmlStatement createDmlStatement(DmlType dmlType, Table table, String textColumnExpression);

public DmlStatement createDmlStatement(DmlType dmlType, String catalogName, String schemaName,
String tableName, Column[] keys, Column[] columns, boolean[] nullKeyValues);
String tableName, Column[] keys, Column[] columns, boolean[] nullKeyValues, String textColumnExpression);

public Object[] getObjectValues(BinaryEncoding encoding, String[] values,
Column[] orderedMetaData);
Expand Down
4 changes: 2 additions & 2 deletions symmetric-db/src/main/java/org/jumpmind/db/platform/mysql/MySqlDmlStatement.java
View file
Expand Up @@ -29,9 +29,9 @@ public class MySqlDmlStatement extends DmlStatement {

public MySqlDmlStatement(DmlType type, String catalogName, String schemaName, String tableName,
Column[] keysColumns, Column[] columns, boolean[] nullKeyValues,
DatabaseInfo databaseInfo, boolean useQuotedIdentifiers) {
DatabaseInfo databaseInfo, boolean useQuotedIdentifiers, String textColumnExpression) {
super(type, catalogName, schemaName, tableName, keysColumns, columns,
nullKeyValues, databaseInfo, useQuotedIdentifiers);
nullKeyValues, databaseInfo, useQuotedIdentifiers, textColumnExpression);
}

@Override
Expand Down
4 changes: 2 additions & 2 deletions symmetric-db/src/main/java/org/jumpmind/db/platform/oracle/OracleDmlStatement.java
View file
Expand Up @@ -31,9 +31,9 @@ public class OracleDmlStatement extends DmlStatement {

public OracleDmlStatement(DmlType type, String catalogName, String schemaName, String tableName,
Column[] keysColumns, Column[] columns, boolean[] nullKeyValues,
DatabaseInfo databaseInfo, boolean useQuotedIdentifiers) {
DatabaseInfo databaseInfo, boolean useQuotedIdentifiers, String textColumnExpression) {
super(type, catalogName, schemaName, tableName, keysColumns, columns,
nullKeyValues, databaseInfo, useQuotedIdentifiers);
nullKeyValues, databaseInfo, useQuotedIdentifiers, textColumnExpression);
}

@Override
Expand Down
4 changes: 2 additions & 2 deletions symmetric-db/src/main/java/org/jumpmind/db/platform/postgresql/PostgreSqlDmlStatement.java
View file
Expand Up @@ -32,9 +32,9 @@ public class PostgreSqlDmlStatement extends DmlStatement {

public PostgreSqlDmlStatement(DmlType type, String catalogName, String schemaName, String tableName,
Column[] keysColumns, Column[] columns, boolean[] nullKeyValues,
DatabaseInfo databaseInfo, boolean useQuotedIdentifiers) {
DatabaseInfo databaseInfo, boolean useQuotedIdentifiers, String textColumnExpression) {
super(type, catalogName, schemaName, tableName, keysColumns, columns,
nullKeyValues, databaseInfo, useQuotedIdentifiers);
nullKeyValues, databaseInfo, useQuotedIdentifiers, textColumnExpression);
}

@Override
Expand Down
4 changes: 2 additions & 2 deletions symmetric-db/src/main/java/org/jumpmind/db/platform/redshift/RedshiftDmlStatement.java
View file
Expand Up @@ -31,9 +31,9 @@ public class RedshiftDmlStatement extends DmlStatement {

public RedshiftDmlStatement(DmlType type, String catalogName, String schemaName, String tableName,
Column[] keysColumns, Column[] columns, boolean[] nullKeyValues,
DatabaseInfo databaseInfo, boolean useQuotedIdentifiers) {
DatabaseInfo databaseInfo, boolean useQuotedIdentifiers, String textColumnExpression) {
super(type, catalogName, schemaName, tableName, keysColumns, columns,
nullKeyValues, databaseInfo, useQuotedIdentifiers);
nullKeyValues, databaseInfo, useQuotedIdentifiers, textColumnExpression);
}

@Override
Expand Down
4 changes: 2 additions & 2 deletions symmetric-db/src/main/java/org/jumpmind/db/platform/sqlanywhere/SqlAnywhereDmlStatement.java
View file
Expand Up @@ -31,9 +31,9 @@ public class SqlAnywhereDmlStatement extends DmlStatement {
public SqlAnywhereDmlStatement(DmlType type, String catalogName,
String schemaName, String tableName, Column[] keysColumns,
Column[] columns, boolean[] nullKeyValues,
DatabaseInfo databaseInfo, boolean useQuotedIdentifiers) {
DatabaseInfo databaseInfo, boolean useQuotedIdentifiers, String textColumnExpression) {
super(type, catalogName, schemaName, tableName, keysColumns, columns,
nullKeyValues, databaseInfo, useQuotedIdentifiers);
nullKeyValues, databaseInfo, useQuotedIdentifiers, textColumnExpression);
}

@Override
Expand Down
4 changes: 2 additions & 2 deletions symmetric-db/src/main/java/org/jumpmind/db/platform/sqlite/SqliteDmlStatement.java
View file
Expand Up @@ -28,9 +28,9 @@ public class SqliteDmlStatement extends DmlStatement {

public SqliteDmlStatement(DmlType type, String catalogName, String schemaName, String tableName,
Column[] keysColumns, Column[] columns, boolean[] nullKeyValues,
DatabaseInfo databaseInfo, boolean useQuotedIdentifiers) {
DatabaseInfo databaseInfo, boolean useQuotedIdentifiers, String textColumnExpression) {
super(type, catalogName, null, tableName, keysColumns, columns,
nullKeyValues, databaseInfo, useQuotedIdentifiers);
nullKeyValues, databaseInfo, useQuotedIdentifiers, textColumnExpression);
}

@Override
Expand Down
27 changes: 21 additions & 6 deletions symmetric-db/src/main/java/org/jumpmind/db/sql/DmlStatement.java
View file
Expand Up @@ -20,6 +20,8 @@
*/
package org.jumpmind.db.sql;

import static org.apache.commons.lang.StringUtils.isNotBlank;

import java.sql.Types;
import java.util.ArrayList;
import java.util.Arrays;
Expand All @@ -33,6 +35,7 @@
import org.apache.commons.lang.NotImplementedException;
import org.jumpmind.db.model.Column;
import org.jumpmind.db.model.Table;
import org.jumpmind.db.model.TypeMap;
import org.jumpmind.db.platform.DatabaseInfo;
import org.jumpmind.db.util.BinaryEncoding;
import org.jumpmind.util.FormatUtils;
Expand Down Expand Up @@ -65,12 +68,15 @@ public enum DmlType {
protected Column[] columns;

protected boolean[] nullKeyValues;


protected String textColumnExpression;

public DmlStatement(DmlType type, String catalogName, String schemaName, String tableName,
Column[] keysColumns, Column[] columns, boolean[] nullKeyValues,
DatabaseInfo databaseInfo, boolean useQuotedIdentifiers) {
DatabaseInfo databaseInfo, boolean useQuotedIdentifiers, String textColumnExpression) {
this.databaseInfo = databaseInfo;
this.columns = columns;
this.textColumnExpression = textColumnExpression;
if (nullKeyValues == null || keysColumns == null
|| nullKeyValues.length != keysColumns.length) {
this.keys = keysColumns;
Expand Down Expand Up @@ -245,7 +251,12 @@ protected void appendColumnEquals(StringBuilder sql, Column[] columns, boolean[]
sql.append(separator);
}
if (!nullColumns[i]) {
sql.append(quote).append(columns[i].getName()).append(quote).append(" = ?");
boolean textType = TypeMap.isTextType(columns[i].getMappedTypeCode());
if (textType && isNotBlank(textColumnExpression)) {
sql.append(quote).append(columns[i].getName()).append(quote).append(" = ").append(textColumnExpression.replace("$(columnName)", "?"));
} else {
sql.append(quote).append(columns[i].getName()).append(quote).append(" = ?");
}
} else {
sql.append(quote).append(columns[i].getName()).append(quote)
.append(" is NULL");
Expand All @@ -271,16 +282,20 @@ protected int appendColumns(StringBuilder sql, Column[] columns, boolean select)
}

protected void appendColumnNameForSql(StringBuilder sql, Column column, boolean select) {
String columnName = column.getName();
String columnName = column.getName();
sql.append(quote).append(columnName).append(quote);
}


protected void appendColumnQuestions(StringBuilder sql, Column[] columns) {
if (columns != null) {
for (int i = 0; i < columns.length; i++) {
if (columns[i] != null) {
sql.append("?").append(",");
boolean textType = TypeMap.isTextType(columns[i].getMappedTypeCode());
if (textType && isNotBlank(textColumnExpression)) {
sql.append(textColumnExpression.replace("$(columnName)", "?")).append(",");
} else {
sql.append("?").append(",");
}
}
}

Expand Down
2 changes: 1 addition & 1 deletion symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/DbExport.java
View file
Expand Up @@ -210,7 +210,7 @@ protected void writeTable(final WriterWrapper writerWrapper, Table table, String

if (!noData) {
if (sql == null) {
sql = platform.createDmlStatement(DmlType.SELECT_ALL, table).getSql();
sql = platform.createDmlStatement(DmlType.SELECT_ALL, table, null).getSql();
}

if (StringUtils.isNotBlank(whereClause)) {
Expand Down
17 changes: 13 additions & 4 deletions symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/DbFill.java
View file
Expand Up @@ -87,6 +87,8 @@ public class DbFill {

private boolean print = false;

private String textColumnExpression;

// Weights given to insert, update, and delete commands when
// randomly selecting a command for any given table.
private int[] dmlWeight = {1,0,0};
Expand Down Expand Up @@ -321,7 +323,7 @@ private Row selectRandomRow(Table table) {
Row row = null;
// Select all rows and return the primary key columns.
String sql = platform.createDmlStatement(DmlType.SELECT_ALL, table.getCatalog(), table.getSchema(), table.getName(),
table.getPrimaryKeyColumns(), table.getColumns(), null).getSql();
table.getPrimaryKeyColumns(), table.getColumns(), null, textColumnExpression).getSql();
final List<Row> rows = new ArrayList<Row>();
platform.getSqlTemplate().query(sql, RANDOM_SELECT_SIZE, new ISqlRowMapper<Object>() {
public Object mapRow(Row row) {
Expand Down Expand Up @@ -623,21 +625,21 @@ public DmlStatement createInsertDmlStatement(Table table) {
return platform.createDmlStatement(DmlType.INSERT,
table.getCatalog(), table.getSchema(), table.getName(),
table.getPrimaryKeyColumns(), table.getColumns(),
null);
null, textColumnExpression);
}

public DmlStatement createUpdateDmlStatement(Table table) {
return platform.createDmlStatement(DmlType.UPDATE,
table.getCatalog(), table.getSchema(), table.getName(),
table.getPrimaryKeyColumns(), table.getNonPrimaryKeyColumns(),
null);
null, textColumnExpression);
}

public DmlStatement createDeleteDmlStatement(Table table) {
return platform.createDmlStatement(DmlType.DELETE,
table.getCatalog(), table.getSchema(), table.getName(),
table.getPrimaryKeyColumns(), table.getNonPrimaryKeyColumns(),
null);
null, textColumnExpression);
}

private Row createRandomInsertValues(DmlStatement updStatement, Table table) {
Expand Down Expand Up @@ -787,4 +789,11 @@ public int getDeleteWeight() {
return dmlWeight[2];
}

public void setTextColumnExpression(String textColumnExpression) {
this.textColumnExpression = textColumnExpression;
}

public String getTextColumnExpression() {
return textColumnExpression;
}
}
9 changes: 9 additions & 0 deletions symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DatabaseWriterSettings.java
View file
Expand Up @@ -54,6 +54,8 @@ public class DatabaseWriterSettings {
protected boolean saveCurrentValueOnError = false;

protected boolean fitToColumn = false;

protected String textColumnExpression;

protected Map<String, Conflict> conflictSettingsByChannel;

Expand Down Expand Up @@ -263,4 +265,11 @@ public boolean isFitToColumn() {
return fitToColumn;
}

public void setTextColumnExpression(String textColumnExpression) {
this.textColumnExpression = textColumnExpression;
}

public String getTextColumnExpression() {
return textColumnExpression;
}
}

0 comments on commit 39b95bb

Please sign in to comment.