Skip to content
Permalink
Browse files
remove deserialization
  • Loading branch information
Ewocker committed Mar 4, 2022
1 parent b056ebb commit fd2b8967350ec76114be867d06e5576ad90b560d
Showing 1 changed file with 14 additions and 14 deletions.
@@ -342,7 +342,7 @@ private File sortStoreFile() throws IOException {
log.info("Proceeding to perform merge of {} sorted files", sortedFiles.size());
Stopwatch w = Stopwatch.createStarted();
File sortedFile = new File(storeDir, getSortedStoreFileName(compressionEnabled));
File serializedSortedFile = new File(storeDir, String.format("serialized-%s", getSortedStoreFileName(false)));
// File serializedSortedFile = new File(storeDir, String.format("serialized-%s", getSortedStoreFileName(false)));

// List<File> inputSortedFilesToMerge = new ArrayList<>(sortedFiles);
// try(BufferedWriter writer = createWriter(sortedFile, compressionEnabled)) {
@@ -364,7 +364,7 @@ private File sortStoreFile() throws IOException {
Collections.addAll(commands, "-T", storeDir.getAbsolutePath());
// Collections.addAll(commands, "-S", "2G");
Collections.addAll(commands, "--parallel", "8");
Collections.addAll(commands, "-o", serializedSortedFile.getAbsolutePath());
Collections.addAll(commands, "-o", sortedFile.getAbsolutePath());
// Collections.addAll(commands, "-t", "/");
// IntStream.range(1, 50).forEach(i -> Collections.addAll(commands, String.format("-k%s,%s", i, i)));
if (compressionEnabled) {
@@ -396,18 +396,18 @@ private File sortStoreFile() throws IOException {
throw new RuntimeException(String.format("Error while running command %s", pb.command()));
}

Stopwatch wDeserialize = Stopwatch.createStarted();
try (BufferedReader reader = FlatFileStoreUtils.createReader(serializedSortedFile, false);
BufferedWriter writer = FlatFileStoreUtils.createWriter(sortedFile, compressionEnabled)) {
String line = reader.readLine();
while (line != null) {
String deserializeLine = entryWriter.deserialize(line);
writer.write(deserializeLine);
writer.newLine();
line = reader.readLine();
}
}
log.info("Deserialize of sorted file completed in {}", wDeserialize);
// Stopwatch wDeserialize = Stopwatch.createStarted();
// try (BufferedReader reader = FlatFileStoreUtils.createReader(serializedSortedFile, false);
// BufferedWriter writer = FlatFileStoreUtils.createWriter(sortedFile, compressionEnabled)) {
// String line = reader.readLine();
// while (line != null) {
// String deserializeLine = entryWriter.deserialize(line);
// writer.write(deserializeLine);
// writer.newLine();
// line = reader.readLine();
// }
// }
// log.info("Deserialize of sorted file completed in {}", wDeserialize);


return sortedFile;

0 comments on commit fd2b896

Please sign in to comment.