diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java index cbca31e260..91ab169f7f 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java @@ -221,6 +221,7 @@ private ParameterConstants() { public final static String DATA_LOADER_TREAT_DATETIME_AS_VARCHAR = "db.treat.date.time.as.varchar.enabled"; public final static String DATA_LOADER_USE_PRIMARY_KEYS_FROM_SOURCE = "dataloader.use.primary.keys.from.source"; public final static String DATA_LOADER_IGNORE_SQL_EVENT_ERRORS = "dataloader.ignore.sql.event.errors"; + public final static String DATA_LOADER_LOG_SQL_PARAMS_ON_ERROR = "dataloader.log.sql.params.on.error"; public final static String DATA_RELOAD_IS_BATCH_INSERT_TRANSACTIONAL = "datareload.batch.insert.transactional"; public final static String DATA_EXTRACTOR_ENABLED = "dataextractor.enable"; public final static String DATA_EXTRACTOR_TEXT_COLUMN_EXPRESSION = "dataextractor.text.column.expression"; diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/load/AbstractDataLoaderFactory.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/load/AbstractDataLoaderFactory.java index f39a8195fe..b59a342c11 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/load/AbstractDataLoaderFactory.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/load/AbstractDataLoaderFactory.java @@ -57,6 +57,7 @@ public DatabaseWriterSettings buildParameterDatabaseWriterSettings(List byChannel = new HashMap(); Map byTable = new HashMap(); boolean multipleDefaultSettingsFound = false; diff --git a/symmetric-core/src/main/resources/symmetric-default.properties b/symmetric-core/src/main/resources/symmetric-default.properties index a2c396bc2e..134e3a36d0 100644 --- a/symmetric-core/src/main/resources/symmetric-default.properties +++ b/symmetric-core/src/main/resources/symmetric-default.properties @@ -1764,6 +1764,13 @@ dataloader.create.table.alter.to.match.db.case=true # Type: boolean dataloader.ignore.sql.event.errors=false +# Indicate that the data loader should log SQL parameter values when a batch fails, which can +# be helpful for debugging. Since SQL parameters will contain application data, some sites +# may need to turn this off for policy compliance. +# DatabaseOverridable: true +# Tags: load +# Type: boolean +dataloader.log.sql.params.on.error=true # If set to true, when a table's schema is sent to the target database default values will not # be included. diff --git a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DatabaseWriterSettings.java b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DatabaseWriterSettings.java index 8fce43c7b3..b7a1f7fca5 100644 --- a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DatabaseWriterSettings.java +++ b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DatabaseWriterSettings.java @@ -46,6 +46,7 @@ public class DatabaseWriterSettings { protected boolean saveCurrentValueOnError = false; protected boolean fitToColumn = false; protected boolean logConflictResolution = false; + protected boolean logSqlParamsOnError = true; protected boolean loadOnlyNode = false; protected String textColumnExpression; protected Map conflictSettingsByChannel; @@ -264,6 +265,14 @@ public boolean isLogConflictResolution() { return logConflictResolution; } + public void setLogSqlParamsOnError(boolean logSqlParamsOnError) { + this.logSqlParamsOnError = logSqlParamsOnError; + } + + public boolean isLogSqlParamsOnError() { + return logSqlParamsOnError; + } + public void setTextColumnExpression(String textColumnExpression) { this.textColumnExpression = textColumnExpression; } diff --git a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DefaultDatabaseWriter.java b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DefaultDatabaseWriter.java index 6843d1d9fe..90b8547cd9 100644 --- a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DefaultDatabaseWriter.java +++ b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DefaultDatabaseWriter.java @@ -792,21 +792,27 @@ protected void logFailureDetails(Throwable e, CsvData data, boolean logLastDmlDe failureMessage.append(batch.getChannelId()); failureMessage.append("'.\n"); if (logLastDmlDetails && currentDmlStatement != null) { - failureMessage.append("Failed sql was: "); - String dynamicSQL = logSqlBuilder.buildDynamicSqlForLog(currentDmlStatement.getSql(), currentDmlValues, currentDmlStatement.getTypes()); - failureMessage.append(dynamicSQL); - if (!dynamicSQL.equals(currentDmlStatement.getSql())) { + boolean shouldLogRawSql = true; + if (writerSettings.isLogSqlParamsOnError()) { + failureMessage.append("Failed sql was: "); + String dynamicSQL = logSqlBuilder.buildDynamicSqlForLog(currentDmlStatement.getSql(), currentDmlValues, currentDmlStatement.getTypes()); + failureMessage.append(dynamicSQL); failureMessage.append("\n"); + shouldLogRawSql = !dynamicSQL.equals(currentDmlStatement.getSql()); + } + if (shouldLogRawSql) { failureMessage.append("Failed raw sql was: "); failureMessage.append(currentDmlStatement.getSql()); } failureMessage.append("\n"); } if (logLastDmlDetails && currentDmlValues != null && currentDmlStatement != null) { - failureMessage.append("Failed sql parameters: "); - failureMessage.append(StringUtils.abbreviate("[" + dmlValuesToString(currentDmlValues, currentDmlStatement.getTypes()) + "]", - CsvData.MAX_DATA_SIZE_TO_PRINT_TO_LOG)); - failureMessage.append("\n"); + if (writerSettings.isLogSqlParamsOnError()) { + failureMessage.append("Failed sql parameters: "); + failureMessage.append(StringUtils.abbreviate("[" + dmlValuesToString(currentDmlValues, currentDmlStatement.getTypes()) + "]", + CsvData.MAX_DATA_SIZE_TO_PRINT_TO_LOG)); + failureMessage.append("\n"); + } failureMessage.append("Failed sql parameters types: "); failureMessage.append("[" + TypeMap.getJdbcTypeDescriptions(currentDmlStatement.getTypes()) + "]"); failureMessage.append("\n"); @@ -820,7 +826,9 @@ protected void logFailureDetails(Throwable e, CsvData data, boolean logLastDmlDe if (e instanceof DataTruncationException) { logDataTruncation(data, failureMessage); } - data.writeCsvDataDetails(failureMessage); + if (writerSettings.isLogSqlParamsOnError()) { + data.writeCsvDataDetails(failureMessage); + } log.info(failureMessage.toString(), e); }