Skip to content
Fetching contributors…
Cannot retrieve contributors at this time
executable file 27 lines (21 sloc) 660 Bytes
"""
Counts how many times each non-excluded word occurs.
"""
class Mapper:
def __init__(self):
file = open(self.params["excludes"],"r")
self.excludes = set(line.strip() for line in file)
file.close()
def __call__(self,key,value):
for word in value.split():
if not (word in self.excludes): yield word,1
def reducer(key,values):
yield key,sum(values)
def runner(job):
job.additer(Mapper,reducer,reducer)
def starter(prog):
excludes = prog.delopt("excludes")
if excludes: prog.addopt("param","excludes="+excludes)
if __name__ == "__main__":
import dumbo
dumbo.main(runner,starter)
Something went wrong with that request. Please try again.