From 974a68afdad771d04b0c79ebbeb554e6d237000f Mon Sep 17 00:00:00 2001 From: "Marko A. Rodriguez" Date: Mon, 30 Sep 2013 10:13:47 -0600 Subject: [PATCH] user can now define their own compression codec in g.getConf(). Fixed #146 --- .../thinkaurelius/faunus/mapreduce/FaunusCompiler.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/thinkaurelius/faunus/mapreduce/FaunusCompiler.java b/src/main/java/com/thinkaurelius/faunus/mapreduce/FaunusCompiler.java index 3a142fd7..43d792c0 100644 --- a/src/main/java/com/thinkaurelius/faunus/mapreduce/FaunusCompiler.java +++ b/src/main/java/com/thinkaurelius/faunus/mapreduce/FaunusCompiler.java @@ -41,6 +41,9 @@ */ public class FaunusCompiler extends Configured implements Tool { + private static final String MAPRED_COMPRESS_MAP_OUTPUT = "mapred.compress.map.output"; + private static final String MAPRED_MAP_OUTPUT_COMPRESSION_CODEC = "mapred.map.output.compression.codec"; + public static final String PATH_ENABLED = Tokens.makeNamespace(FaunusCompiler.class) + ".pathEnabled"; public static final String TESTING = Tokens.makeNamespace(FaunusCompiler.class) + ".testing"; public static final Logger logger = Logger.getLogger(FaunusCompiler.class); @@ -171,8 +174,10 @@ public void completeSequence() { if (null != this.combinerClass) job.setCombinerClass(this.combinerClass); // if there is a reduce task, compress the map output to limit network traffic - job.getConfiguration().setBoolean("mapred.compress.map.output", true); - job.getConfiguration().setClass("mapred.map.output.compression.codec", DefaultCodec.class, CompressionCodec.class); + if (null == job.getConfiguration().get(MAPRED_COMPRESS_MAP_OUTPUT, null)) + job.getConfiguration().setBoolean(MAPRED_COMPRESS_MAP_OUTPUT, true); + if (null == job.getConfiguration().get(MAPRED_MAP_OUTPUT_COMPRESSION_CODEC, null)) + job.getConfiguration().setClass(MAPRED_MAP_OUTPUT_COMPRESSION_CODEC, DefaultCodec.class, CompressionCodec.class); } else { job.setNumReduceTasks(0); }