Skip to content

Commit

Permalink
Make examples a little more interesting by counting occurences of types
Browse files Browse the repository at this point in the history
and properties
  • Loading branch information
tfmorris committed Nov 15, 2012
1 parent 7b37f53 commit f4628b4
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 15 deletions.
13 changes: 5 additions & 8 deletions src/main/java/org/hackreduce/examples/freebase/QuadCounter.java
Expand Up @@ -19,9 +19,9 @@


/**
* This MapReduce job will count the total number of assertions in the Freebase
* quad dump.
*
* This MapReduce job will count the total number of assertions for each
* property in the Freebase quad dump as well as the number of quads with
* each of the different destination value types (e.g. key, another object, etc)
*/
public class QuadCounter extends org.hackreduce.examples.RecordCounter {

Expand All @@ -32,10 +32,6 @@ public enum Count {

public static class RecordCounterMapper extends FreebaseQuadMapper<Text, LongWritable> {

// Our own made up key to send all counts to a single Reducer, so we can
// aggregate a total value.
public static final Text TOTAL_COUNT = new Text("total");

// Just to save on object instantiation
public static final LongWritable ONE_COUNT = new LongWritable(1);

Expand All @@ -44,7 +40,8 @@ protected void map(FreebaseQuadRecord record, Context context) throws IOExceptio
InterruptedException {

context.getCounter(Count.TOTAL_RECORDS).increment(1);
context.write(TOTAL_COUNT, ONE_COUNT);
context.write(new Text(record.getProperty()), ONE_COUNT);
context.write(new Text(record.getType().toString()), ONE_COUNT);
}

}
Expand Down
12 changes: 5 additions & 7 deletions src/main/java/org/hackreduce/examples/freebase/TopicCounter.java
Expand Up @@ -29,21 +29,19 @@ public enum Count {

public static class RecordCounterMapper extends FreebaseTopicMapper<Text, LongWritable> {

// Our own made up key to send all counts to a single Reducer, so we can
// aggregate a total value.
public static final Text TOTAL_COUNT = new Text("total");

// Just to save on object instantiation
public static final LongWritable ONE_COUNT = new LongWritable(1);

@Override
protected void map(FreebaseTopicRecord record, Context context) throws IOException,
InterruptedException {

context.getCounter(Count.TOTAL_RECORDS).increment(1);
context.write(TOTAL_COUNT, ONE_COUNT);
context.getCounter(Count.TOTAL_RECORDS).increment(1);
// Emit a record for each type the topic has
for (String t : record.getFb_types()) {
context.write(new Text(t), ONE_COUNT);
}
}

}

@Override
Expand Down

0 comments on commit f4628b4

Please sign in to comment.