Permalink
Browse files

log sizes of created smoosh files (#3817)

* log when merging of intermediate segments starts during batch ingestion

* log sizes of created smoosh files
  • Loading branch information...
1 parent 220ca7e commit 7ced0e8759143bf35f598f0a48c192bb21e5c6be @himanshug himanshug committed with fjy Jan 5, 2017
@@ -695,9 +695,16 @@ public void doRun()
for (File file : toMerge) {
indexes.add(HadoopDruidIndexerConfig.INDEX_IO.loadIndex(file));
}
+
+ log.info("starting merge of intermediate persisted segments.");
+ long mergeStartTime = System.currentTimeMillis();
mergedBase = mergeQueryableIndex(
indexes, aggregators, new File(baseFlushFile, "merged"), progressIndicator
);
+ log.info(
+ "finished merge of intermediate persisted segments. time taken [%d] ms.",
+ (System.currentTimeMillis() - mergeStartTime)
+ );
}
final FileSystem outputFS = new Path(config.getSchema().getIOConfig().getSegmentOutputPath())
.getFileSystem(context.getConfiguration());
@@ -414,25 +414,27 @@ private Outer getNewCurrOut() throws FileNotFoundException
final int fileNum = outFiles.size();
File outFile = makeChunkFile(baseDir, fileNum);
outFiles.add(outFile);
- return new Outer(fileNum, new FileOutputStream(outFile), maxChunkSize);
+ return new Outer(fileNum, outFile, maxChunkSize);
}
public static class Outer implements SmooshedWriter
{
private final int fileNum;
private final int maxLength;
+ private final File outFile;
private final GatheringByteChannel channel;
private final Closer closer = Closer.create();
private int currOffset = 0;
- Outer(int fileNum, FileOutputStream output, int maxLength)
+ Outer(int fileNum, File outFile, int maxLength) throws FileNotFoundException
{
this.fileNum = fileNum;
- this.channel = output.getChannel();
+ this.outFile = outFile;
this.maxLength = maxLength;
- closer.register(output);
- closer.register(channel);
+
+ FileOutputStream outStream = closer.register(new FileOutputStream(outFile));
+ this.channel = closer.register(outStream.getChannel());
}
public int getFileNum()
@@ -494,6 +496,7 @@ public boolean isOpen()
public void close() throws IOException
{
closer.close();
+ FileSmoosher.LOG.info("Created smoosh file [%s] of size [%s] bytes.", outFile.getAbsolutePath(), outFile.length());
}
}
}

0 comments on commit 7ced0e8

Please sign in to comment.