0003341: Improve PostgreSQL bulk loader to tolerate NULL bytes in

incoming data
JumpMind · Dec 20, 2017 · 515e3eb · 515e3eb
1 parent 1131a09
commit 515e3eb
Showing 1 changed file with 11 additions and 0 deletions.
diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/io/PostgresBulkDatabaseWriter.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/io/PostgresBulkDatabaseWriter.java
@@ -103,6 +103,7 @@ protected void bulkWrite(CsvData data) {
                             }
                         }
                         String formattedData = CsvUtils.escapeCsvData(parsedData, '\n', '\'', CsvWriter.ESCAPE_MODE_DOUBLED);
+                        formattedData = removeIllegalCharacters(formattedData);
                         byte[] dataToLoad = formattedData.getBytes();
                         copyIn.writeToCopy(dataToLoad, 0, dataToLoad.length);
                         loadedRows++;
@@ -130,6 +131,16 @@ protected void bulkWrite(CsvData data) {
         statistics.get(batch).increment(DataWriterStatisticConstants.LINENUMBER);
     }
 
+    protected String removeIllegalCharacters(String formattedData) {
+        StringBuilder buff = new StringBuilder(formattedData.length());
+        for (char c : formattedData.toCharArray()) {
+            if (c > 0) {
+                buff.append(c);
+            }
+        }
+        return buff.toString();
+    }
+
     protected void flush() {
         if (copyIn != null) {
             try {