Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

executable file 61 lines (53 sloc) 1.285 kb
#!/usr/bin/env ruby
$: << File.dirname(__FILE__)+'/../lib'
require 'wukong'
module Size
#
# Feed the entire dataset through wc and sum the results
#
class Script < Wukong::Script
#
# Don't implement a wukong script to do something if there's a unix command
# that does it faster: just override map_command or reduce_command in your
# subclass of Wukong::Script to return the complete command line
#
def map_command
'/usr/bin/wc'
end
# Make all records go to one reducer
def default_options
super.merge :reduce_tasks => 1
end
end
#
# Sums the numeric value of each column in its input
#
class Reducer < Wukong::Streamer::Base
attr_accessor :sums
#
# The unix +wc+ command uses whitespace, not tabs, so we'll recordize
# accordingly.
#
def recordize line
line.strip.split(/\s+/)
end
#
# add each corresponding column in the input
#
def process *vals
self.sums = vals.zip( sums || [] ).map{|val,sum| val.to_i + sum.to_i }
end
#
# run through the whole reduction input and then output the total
#
def stream *args
super *args
emit sums
end
end
end
# Execute the script
Size::Script.new(
nil,
Size::Reducer
).run
Jump to Line
Something went wrong with that request. Please try again.