Skip to content

Commit

Permalink
Use NullMemoryManager in OrcFileWriter
Browse files Browse the repository at this point in the history
The MemoryManager in the OrcFileWriter causes writes to be slow as all
writers contend on the MemoryManager. We do not need the MemoryManager
for Raptor, so replace it with a NullMemoryManager
  • Loading branch information
nileema committed Sep 15, 2015
1 parent 518839b commit d94f575
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 2 deletions.
2 changes: 1 addition & 1 deletion pom.xml
Expand Up @@ -250,7 +250,7 @@
<dependency>
<groupId>com.facebook.presto.hive</groupId>
<artifactId>hive-apache</artifactId>
<version>0.14</version>
<version>0.15</version>
</dependency>

<dependency>
Expand Down
Expand Up @@ -27,9 +27,11 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.orc.NullMemoryManager;
import org.apache.hadoop.hive.ql.io.orc.OrcFile;
import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat;
import org.apache.hadoop.hive.ql.io.orc.OrcSerde;
import org.apache.hadoop.hive.ql.io.orc.OrcWriterOptions;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector;
Expand Down Expand Up @@ -180,9 +182,11 @@ private static RecordWriter createRecordWriter(Path target, Configuration conf)
{
try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader());
FileSystem fileSystem = new SyncingFileSystem(CONFIGURATION)) {
OrcFile.WriterOptions options = OrcFile.writerOptions(conf)
OrcFile.WriterOptions options = new OrcWriterOptions(conf)
.memory(new NullMemoryManager(conf))
.fileSystem(fileSystem)
.compress(SNAPPY);

return WRITER_CONSTRUCTOR.newInstance(target, options);
}
catch (ReflectiveOperationException | IOException e) {
Expand Down

0 comments on commit d94f575

Please sign in to comment.