Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

MAPREDUCE-5808. Port output replication factor configurable for teras…

…ort to Hadoop 1.x. Contributed by Chuan Liu.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1-win@1580988 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information...
commit 08318f9a9d31c24b1c9b81d6c5c318def14af10b 1 parent e1c4bde
@cnauroth cnauroth authored
View
3  CHANGES.branch-1-win.txt
@@ -513,3 +513,6 @@ Branch-hadoop-1-win (branched from branch-1 at r1293509 on 2012-02-24 17:46:42)
HDFS-5516. WebHDFS does not require user name when anonymous http requests
are disallowed. (Miodrag Radulovic via cnauroth)
+
+ MAPREDUCE-5808. Port output replication factor configurable for terasort to
+ Hadoop 1.x. (Chuan Liu via cnauroth)
View
7 src/examples/org/apache/hadoop/examples/terasort/TeraSort.java
@@ -49,6 +49,7 @@
*/
public class TeraSort extends Configured implements Tool {
private static final Log LOG = LogFactory.getLog(TeraSort.class);
+ private static final String OUTPUT_REPLICATION = "mapreduce.terasort.output.replication";
/**
* A partitioner that splits text keys into roughly equal partitions
@@ -222,6 +223,10 @@ public int getPartition(Text key, Text value, int numPartitions) {
}
}
+
+ public static int getOutputReplication(JobConf job) {
+ return job.getInt(OUTPUT_REPLICATION, 1);
+ }
public int run(String[] args) throws Exception {
LOG.info("starting");
@@ -243,7 +248,7 @@ public int run(String[] args) throws Exception {
TeraInputFormat.writePartitionFile(job, partitionFile);
DistributedCache.addCacheFile(partitionUri, job);
DistributedCache.createSymlink(job);
- job.setInt("dfs.replication", 1);
+ job.setInt("dfs.replication", getOutputReplication(job));
TeraOutputFormat.setFinalSync(job, true);
JobClient.runJob(job);
LOG.info("done");
Please sign in to comment.
Something went wrong with that request. Please try again.