Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
CHUKWA-647. Spread out intermediate data with the same ReduceType int…
…o different Reduce Tasks. Contributed by Jie Huang.

git-svn-id: https://svn.apache.org/repos/asf/incubator/chukwa/trunk@1362318 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information
Ariel Shemaiah Rabkin committed Jul 17, 2012
1 parent 7d1290c commit e8126396d3deab11187beebdc5ccd159227bb1ee
Showing 2 changed files with 6 additions and 1 deletion.
@@ -64,5 +64,6 @@ public interface CHUKWA_CONSTANT {
public static final String ARCHIVES_IN_ERROR_DIR_NAME = "inError/";

public static final String POST_DEMUX_DATA_LOADER = "chukwa.post.demux.data.loader";
public static final String INCLUDE_KEY_IN_PARTITIONER = "_";
public static final String POST_DEMUX_SUCCESS_ACTION = "chukwa.post.demux.success.action";
}
@@ -21,6 +21,8 @@

import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord;
import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey;
import org.apache.hadoop.chukwa.extraction.CHUKWA_CONSTANT;

import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Partitioner;
import org.apache.log4j.Logger;
@@ -44,7 +46,9 @@ public int getPartition(
+ "] - Reducer:"
+ ((key.getReduceType().hashCode() & Integer.MAX_VALUE) % numReduceTasks));
}
return (key.getReduceType().hashCode() & Integer.MAX_VALUE)
String hashkey = key.getReduceType();
if(key.getKey().startsWith(CHUKWA_CONSTANT.INCLUDE_KEY_IN_PARTITIONER)) hashkey = key.getReduceType()+"#"+key.getKey();
return (hashkey.hashCode() & Integer.MAX_VALUE)
% numReduceTasks;
}

0 comments on commit e812639

Please sign in to comment.