Example programs

They don’t illustrate all of Dumbo’s nifty features, but you should be able to get the basics from these examples (read dumbo.py to get familiar with the rest!):

simplewordcount.py

def mapper(key,value):
    for word in value.split(): yield word,1

def reducer(key,values):
    yield key,sum(values)

if __name__ == "__main__":
    import dumbo
    dumbo.run(mapper,reducer)

wordcount.py

excludes = None

def loadexcludes():
    global excludes
    file = open("excludes.txt","r")
    excludes = set(line.strip() for line in file)
    file.close()

def mapper(key,value):
    for word in value.split():
        if not (word in excludes): yield word,1

def reducer(key,values):
    yield key,sum(values)

if __name__ == "__main__":
    import dumbo
    dumbo.run(mapper,reducer,combiner=reducer,mapconf=loadexcludes)

oowordcount.py

class Mapper:
    def conf(self):
        file = open("excludes.txt","r")
        self.excludes = set(line.strip() for line in file)
        file.close()

    def map(key,value):
        for word in value.split():
            if not (word in self.excludes): yield word,1

def reducer(key,values):
    yield key,sum(values)

if __name__ == "__main__":
    import dumbo
    mapper = Mapper()
    dumbo.run(mapper.map,reducer,combiner=reducer,mapconf=mapper.conf)

itertwice.py

def mapper1(key,value):
    for word in value.split(): yield word,1

def mapper2(key,value):
    for letter in value.split()[0]: yield letter,1

def reducer1(key,values):
    count = sum(values)
    if count > 1: yield key,count

def reducer2(key,values):
    yield key,sum(values)

if __name__ == "__main__":
    import dumbo
    job = dumbo.Job()
    job.additer(mapper1,reducer1,reducer2)
    job.additer(mapper2,reducer2,reducer2)
    job.run()

greplogs.py

def mapper(key,value):
    if value.find("playground.last.fm") >= 0: yield value,

if __name__ == "__main__":
    import dumbo
    dumbo.run(mapper)

sumnorm.py

def mapper(key,value):
    parts = value.split()
    yield parts[0],(parts[1:-1],float(parts[-1]))

def reducer(key,values):
    values = list(values)
    s = sum(map(lambda x: x[1],values))
    for value in values:
        yield [key]+value[0],int(value[1]*100/s)

if __name__ == "__main__":
    import dumbo
    dumbo.run(mapper,reducer)