0001926: Add dataloader.fit.to.column that when set will truncate dat…

…a to fit in the target column

symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java

@@ -146,6 +146,7 @@ private ParameterConstants() {
     public final static String INCOMING_BATCH_RECORD_OK_ENABLED = "incoming.batches.record.ok.enabled";
     public final static String DATA_LOADER_ENABLED = "dataloader.enable";
     public final static String DATA_LOADER_IGNORE_MISSING_TABLES = "dataloader.ignore.missing.tables";
+    public final static String DATA_LOADER_FIT_TO_COLUMN = "dataloader.fit.to.column";
     public final static String DATA_LOADER_ERROR_RECORD_CUR_VAL = "dataloader.error.save.curval";
     public final static String DATA_LOADER_NUM_OF_ACK_RETRIES = "num.of.ack.retries";
     public final static String DATA_LOADER_TIME_BETWEEN_ACK_RETRIES = "time.between.ack.retries.ms";

symmetric-core/src/main/java/org/jumpmind/symmetric/load/DefaultDataLoaderFactory.java

@@ -116,6 +116,7 @@ protected DatabaseWriterSettings buildDatabaseWriterSettings(
                 .is(ParameterConstants.DATA_LOADER_TREAT_DATETIME_AS_VARCHAR));
         settings.setSaveCurrentValueOnError(parameterService.is(
                 ParameterConstants.DATA_LOADER_ERROR_RECORD_CUR_VAL, false));
+        settings.setFitToColumn(parameterService.is(ParameterConstants.DATA_LOADER_FIT_TO_COLUMN, false));
 
         Map<String, Conflict> byChannel = new HashMap<String, Conflict>();
         Map<String, Conflict> byTable = new HashMap<String, Conflict>();

symmetric-core/src/main/resources/symmetric-default.properties

@@ -932,6 +932,14 @@ dataloader.enable=true
 # Type: boolean
 dataloader.ignore.missing.tables=false
 
+# Indicate that the data loader should truncate data that is bigger than the target columns can handle.  This applies 
+# to text-based columns only.  
+#
+# DatabaseOverridable: true
+# Tags: load
+# Type: boolean
+dataloader.fit.to.column=false
+
 # This is the maximum number of rows that will be supported in a
 # single transaction.  If the database transaction row count reaches a size
 # that is greater than this number then the transaction will be auto committed.

symmetric-db/src/main/java/org/jumpmind/db/platform/AbstractDatabasePlatform.java

@@ -334,26 +334,26 @@ public Object[] getObjectValues(BinaryEncoding encoding, Table table, String[] c
     }
 
     public Object[] getObjectValues(BinaryEncoding encoding, Table table, String[] columnNames,
-            String[] values, boolean useVariableDates) {
+            String[] values, boolean useVariableDates, boolean fitToColumn) {
         Column[] metaData = Table.orderColumns(columnNames, table);
-        return getObjectValues(encoding, values, metaData, useVariableDates);
+        return getObjectValues(encoding, values, metaData, useVariableDates, fitToColumn);
     }
 
     public Object[] getObjectValues(BinaryEncoding encoding, String[] values,
             Column[] orderedMetaData) {
-        return getObjectValues(encoding, values, orderedMetaData, false);
+        return getObjectValues(encoding, values, orderedMetaData, false, false);
     }
 
     public Object[] getObjectValues(BinaryEncoding encoding, String[] values,
-            Column[] orderedMetaData, boolean useVariableDates) {
+            Column[] orderedMetaData, boolean useVariableDates, boolean fitToColumn) {
         if (values != null) {
             List<Object> list = new ArrayList<Object>(values.length);
             for (int i = 0; i < values.length; i++) {
                 String value = values[i];
                 Column column = orderedMetaData.length > i ? orderedMetaData[i] : null;
                 try {
                     if (column != null) {
-                        list.add(getObjectValue(value, column, encoding, useVariableDates));
+                        list.add(getObjectValue(value, column, encoding, useVariableDates, fitToColumn));
                     }
                 } catch (Exception ex) {
                     String valueTrimmed = FormatUtils.abbreviateForLogging(value);
@@ -371,7 +371,7 @@ public Object[] getObjectValues(BinaryEncoding encoding, String[] values,
     }
 
     protected Object getObjectValue(String value, Column column, BinaryEncoding encoding,
-            boolean useVariableDates) throws DecoderException {
+            boolean useVariableDates, boolean fitToColumn) throws DecoderException {
         Object objectValue = value;
         int type = column.getMappedTypeCode();
         if ((value == null || (getDdlBuilder().getDatabaseInfo().isEmptyStringNulled() && value
@@ -417,7 +417,12 @@ protected Object getObjectValue(String value, Column column, BinaryEncoding enco
             }
         }
         if (objectValue instanceof String) {
-            objectValue = cleanTextForTextBasedColumns((String) objectValue);
+            String stringValue = cleanTextForTextBasedColumns((String) objectValue);
+            int size = column.getSizeAsInt();
+            if (fitToColumn && size > 0 && stringValue.length() > size) {
+                stringValue = stringValue.substring(0, size);
+            }
+            objectValue = stringValue;
         }
 
         return objectValue;
@@ -430,28 +435,29 @@ protected Object parseBigDecimal(String value) {
          * point, but we need a period
          */
         value = cleanNumber(value);
-        return new BigDecimal(value.replace(',', '.').trim());
+        return new BigDecimal(value.replace(',', '.'));
     }    
 
     protected Object parseBigInteger(String value) {
         try {
             value = cleanNumber(value);
             return new Long(value.trim());
         } catch (NumberFormatException ex) {            
-            return new BigInteger(value.trim());        
+            return new BigInteger(value);        
         }
     }    
 
     protected Object parseInteger(String value) {
         try {
             value = cleanNumber(value);
-            return Integer.parseInt(value.trim());
+            return Integer.parseInt(value);
         } catch (NumberFormatException ex) {
-            return new BigInteger(value.trim());        
+            return new BigInteger(value);        
         }
     }
 
     protected String cleanNumber(String value) {
+        value = value.trim();
         if (value.equalsIgnoreCase("true")) {
             return "1";
         } else if (value.equalsIgnoreCase("false")) {

symmetric-db/src/main/java/org/jumpmind/db/platform/IDatabasePlatform.java

@@ -124,10 +124,10 @@ public Object[] getObjectValues(BinaryEncoding encoding, Table table, String[] c
             String[] values);
 
     public Object[] getObjectValues(BinaryEncoding encoding, Table table, String[] columnNames,
-            String[] values, boolean useVariableDates);
+            String[] values, boolean useVariableDates, boolean fitToColumn);
 
     public Object[] getObjectValues(BinaryEncoding encoding, String[] values,
-            Column[] orderedMetaData, boolean useVariableDates);
+            Column[] orderedMetaData, boolean useVariableDates, boolean fitToColumn);
 
     public String[] getStringValues(BinaryEncoding encoding, Column[] metaData, Row row, boolean useVariableDates);
 

symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DatabaseWriterSettings.java

@@ -52,14 +52,16 @@ public class DatabaseWriterSettings {
     protected boolean ignoreMissingTables = true;
 
     protected boolean saveCurrentValueOnError = false;
+
+    protected boolean fitToColumn = false;
 
     protected Map<String, Conflict> conflictSettingsByChannel;
 
     protected Map<String, Conflict> conflictSettingsByTable;
 
     protected List<IDatabaseWriterFilter> databaseWriterFilters;
 
-    protected List<IDatabaseWriterErrorHandler> databaseWriterErrorHandlers;
+    protected List<IDatabaseWriterErrorHandler> databaseWriterErrorHandlers;        
 
     protected List<ResolvedData> resolvedData;
 
@@ -252,5 +254,13 @@ public boolean isSaveCurrentValueOnError() {
     public void setSaveCurrentValueOnError(boolean saveCurrentValueOnError) {
         this.saveCurrentValueOnError = saveCurrentValueOnError;
     }
+
+    public void setFitToColumn(boolean fitToColumn) {
+        this.fitToColumn = fitToColumn;
+    }
+
+    public boolean isFitToColumn() {
+        return fitToColumn;
+    }
 
 }

symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DefaultDatabaseWriter.java

@@ -780,7 +780,7 @@ protected boolean doesColumnNeedUpdated(int columnIndex, Column column, CsvData
 
     protected int execute(CsvData data, String[] values) {
         Object[] objectValues = platform.getObjectValues(batch.getBinaryEncoding(), values,
-                currentDmlStatement.getMetaData());
+                currentDmlStatement.getMetaData(), false, writerSettings.isFitToColumn());
         if (log.isDebugEnabled()) {
             log.debug("Submitting data {} with types {}", Arrays.toString(objectValues),
                     Arrays.toString(this.currentDmlStatement.getTypes()));

symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/StructureDataWriter.java

@@ -162,7 +162,7 @@ protected String buildSql(DmlType dmlType, String[] values, Column[] columns) {
         // TODO support primary key updates
         DmlStatement statement = DmlStatementFactory.createDmlStatement(targetDatabaseName,
                 dmlType, currentTable, useQuotedIdentifiers);
-        Object[] objects = platform.getObjectValues(binaryEncoding, values, columns, false);
+        Object[] objects = platform.getObjectValues(binaryEncoding, values, columns, false, false);
         Row row = new Row(columns.length);
         for (int i = 0; i < columns.length; i++) {
             row.put(columns[i].getName(), objects[i]);