Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
0003341: Improve PostgreSQL bulk loader to tolerate NULL bytes in
incoming data
  • Loading branch information
mmichalek committed Dec 20, 2017
1 parent 1131a09 commit 515e3eb
Showing 1 changed file with 11 additions and 0 deletions.
Expand Up @@ -103,6 +103,7 @@ protected void bulkWrite(CsvData data) {
}
}
String formattedData = CsvUtils.escapeCsvData(parsedData, '\n', '\'', CsvWriter.ESCAPE_MODE_DOUBLED);
formattedData = removeIllegalCharacters(formattedData);
byte[] dataToLoad = formattedData.getBytes();
copyIn.writeToCopy(dataToLoad, 0, dataToLoad.length);
loadedRows++;
Expand Down Expand Up @@ -130,6 +131,16 @@ protected void bulkWrite(CsvData data) {
statistics.get(batch).increment(DataWriterStatisticConstants.LINENUMBER);
}

protected String removeIllegalCharacters(String formattedData) {
StringBuilder buff = new StringBuilder(formattedData.length());
for (char c : formattedData.toCharArray()) {
if (c > 0) {
buff.append(c);
}
}
return buff.toString();
}

protected void flush() {
if (copyIn != null) {
try {
Expand Down

0 comments on commit 515e3eb

Please sign in to comment.