0005376: Parameter to disable logging of application data from a batch

error
JumpMind · Jul 26, 2022 · f9f23b4 · f9f23b4
1 parent ecedd68
commit f9f23b4
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 9 deletions.
diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java
@@ -221,6 +221,7 @@ private ParameterConstants() {
     public final static String DATA_LOADER_TREAT_DATETIME_AS_VARCHAR = "db.treat.date.time.as.varchar.enabled";
     public final static String DATA_LOADER_USE_PRIMARY_KEYS_FROM_SOURCE = "dataloader.use.primary.keys.from.source";
     public final static String DATA_LOADER_IGNORE_SQL_EVENT_ERRORS = "dataloader.ignore.sql.event.errors";
+    public final static String DATA_LOADER_LOG_SQL_PARAMS_ON_ERROR = "dataloader.log.sql.params.on.error";
     public final static String DATA_RELOAD_IS_BATCH_INSERT_TRANSACTIONAL = "datareload.batch.insert.transactional";
     public final static String DATA_EXTRACTOR_ENABLED = "dataextractor.enable";
     public final static String DATA_EXTRACTOR_TEXT_COLUMN_EXPRESSION = "dataextractor.text.column.expression";

diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/load/AbstractDataLoaderFactory.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/load/AbstractDataLoaderFactory.java
@@ -57,6 +57,7 @@ public DatabaseWriterSettings buildParameterDatabaseWriterSettings(List<? extend
         settings.setUsePrimaryKeysFromSource(
                 parameterService.is(ParameterConstants.DATA_LOADER_USE_PRIMARY_KEYS_FROM_SOURCE));
         settings.setIgnoreSqlDataEventFailures(parameterService.is(ParameterConstants.DATA_LOADER_IGNORE_SQL_EVENT_ERRORS, false));
+        settings.setLogSqlParamsOnError(parameterService.is(ParameterConstants.DATA_LOADER_LOG_SQL_PARAMS_ON_ERROR, true));
         Map<String, Conflict> byChannel = new HashMap<String, Conflict>();
         Map<String, Conflict> byTable = new HashMap<String, Conflict>();
         boolean multipleDefaultSettingsFound = false;

diff --git a/symmetric-core/src/main/resources/symmetric-default.properties b/symmetric-core/src/main/resources/symmetric-default.properties
@@ -1764,6 +1764,13 @@ dataloader.create.table.alter.to.match.db.case=true
 # Type: boolean
 dataloader.ignore.sql.event.errors=false
 
+# Indicate that the data loader should log SQL parameter values when a batch fails, which can
+# be helpful for debugging.  Since SQL parameters will contain application data, some sites
+# may need to turn this off for policy compliance.
+# DatabaseOverridable: true
+# Tags: load
+# Type: boolean
+dataloader.log.sql.params.on.error=true
 
 # If set to true, when a table's schema is sent to the target database default values will not 
 # be included.

diff --git a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DatabaseWriterSettings.java b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DatabaseWriterSettings.java
@@ -46,6 +46,7 @@ public class DatabaseWriterSettings {
     protected boolean saveCurrentValueOnError = false;
     protected boolean fitToColumn = false;
     protected boolean logConflictResolution = false;
+    protected boolean logSqlParamsOnError = true;
     protected boolean loadOnlyNode = false;
     protected String textColumnExpression;
     protected Map<String, Conflict> conflictSettingsByChannel;
@@ -264,6 +265,14 @@ public boolean isLogConflictResolution() {
         return logConflictResolution;
     }
 
+    public void setLogSqlParamsOnError(boolean logSqlParamsOnError) {
+        this.logSqlParamsOnError = logSqlParamsOnError;
+    }
+
+    public boolean isLogSqlParamsOnError() {
+        return logSqlParamsOnError;
+    }
+
     public void setTextColumnExpression(String textColumnExpression) {
         this.textColumnExpression = textColumnExpression;
     }

diff --git a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DefaultDatabaseWriter.java b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DefaultDatabaseWriter.java
@@ -792,21 +792,27 @@ protected void logFailureDetails(Throwable e, CsvData data, boolean logLastDmlDe
         failureMessage.append(batch.getChannelId());
         failureMessage.append("'.\n");
         if (logLastDmlDetails && currentDmlStatement != null) {
-            failureMessage.append("Failed sql was: ");
-            String dynamicSQL = logSqlBuilder.buildDynamicSqlForLog(currentDmlStatement.getSql(), currentDmlValues, currentDmlStatement.getTypes());
-            failureMessage.append(dynamicSQL);
-            if (!dynamicSQL.equals(currentDmlStatement.getSql())) {
+            boolean shouldLogRawSql = true;
+            if (writerSettings.isLogSqlParamsOnError()) {
+                failureMessage.append("Failed sql was: ");
+                String dynamicSQL = logSqlBuilder.buildDynamicSqlForLog(currentDmlStatement.getSql(), currentDmlValues, currentDmlStatement.getTypes());
+                failureMessage.append(dynamicSQL);
                 failureMessage.append("\n");
+                shouldLogRawSql = !dynamicSQL.equals(currentDmlStatement.getSql());
+            }
+            if (shouldLogRawSql) {
                 failureMessage.append("Failed raw sql was: ");
                 failureMessage.append(currentDmlStatement.getSql());
             }
             failureMessage.append("\n");
         }
         if (logLastDmlDetails && currentDmlValues != null && currentDmlStatement != null) {
-            failureMessage.append("Failed sql parameters: ");
-            failureMessage.append(StringUtils.abbreviate("[" + dmlValuesToString(currentDmlValues, currentDmlStatement.getTypes()) + "]",
-                    CsvData.MAX_DATA_SIZE_TO_PRINT_TO_LOG));
-            failureMessage.append("\n");
+            if (writerSettings.isLogSqlParamsOnError()) {
+                failureMessage.append("Failed sql parameters: ");
+                failureMessage.append(StringUtils.abbreviate("[" + dmlValuesToString(currentDmlValues, currentDmlStatement.getTypes()) + "]",
+                        CsvData.MAX_DATA_SIZE_TO_PRINT_TO_LOG));
+                failureMessage.append("\n");
+            }
             failureMessage.append("Failed sql parameters types: ");
             failureMessage.append("[" + TypeMap.getJdbcTypeDescriptions(currentDmlStatement.getTypes()) + "]");
             failureMessage.append("\n");
@@ -820,7 +826,9 @@ protected void logFailureDetails(Throwable e, CsvData data, boolean logLastDmlDe
         if (e instanceof DataTruncationException) {
             logDataTruncation(data, failureMessage);
         }
-        data.writeCsvDataDetails(failureMessage);
+        if (writerSettings.isLogSqlParamsOnError()) {
+            data.writeCsvDataDetails(failureMessage);
+        }
         log.info(failureMessage.toString(), e);
     }