Skip to content

Commit

Permalink
0004719: MultiBatchStagingWriter writes "\r\n" on Windows for end of
Browse files Browse the repository at this point in the history
lines in child batch staging files
  • Loading branch information
Philip Marzullo committed Dec 23, 2020
1 parent c91a588 commit da8e02d
Showing 1 changed file with 20 additions and 6 deletions.
Expand Up @@ -245,14 +245,21 @@ protected void checkSendChildRequests(OutgoingBatch parentBatch, IStagedResource
log.debug("About to copy batch {} to batch {}-{}", this.outgoingBatch.getNodeBatchId(), childRequest.getNodeId(), childBatchId);
BufferedReader reader = parentResource.getReader();
BufferedWriter writer = childResource.getWriter(memoryThresholdInBytes);
String line = null;
try {
while ((line = reader.readLine()) != null) {
if (line.startsWith(CsvConstants.BATCH) || line.startsWith(CsvConstants.COMMIT)) {
line = line.replace(Long.toString(outgoingBatch.getBatchId()), Long.toString(childBatchId));
StringBuffer sb = new StringBuffer();
int i;
while ((i = reader.read()) != -1) {
char c = (char) i;
sb.append(c);
if (c == '\n') {
String s = replaceBatchId(sb.toString(), outgoingBatch.getBatchId(), childBatchId);
writer.write(s);
sb.delete(0, sb.length());
}
writer.write(line);
writer.write(System.lineSeparator());
}
if (sb.length() > 0) {
String s = replaceBatchId(sb.toString(), outgoingBatch.getBatchId(), childBatchId);
writer.write(s);
}
} catch (Exception e) {
throw new RuntimeException("Failed to copy batch " + this.outgoingBatch.getNodeBatchId() + " to batch " +
Expand Down Expand Up @@ -281,6 +288,13 @@ protected void checkSendChildRequests(OutgoingBatch parentBatch, IStagedResource
}
}
}

private static String replaceBatchId(String s, long originalBatchId, long newBatchId) {
if (s.startsWith(CsvConstants.BATCH) || s.startsWith(CsvConstants.COMMIT)) {
s = s.replace(Long.toString(originalBatchId), Long.toString(newBatchId));
}
return s;
}

@Override
public void end(Table table) {
Expand Down

0 comments on commit da8e02d

Please sign in to comment.