Skip to content

Commit

Permalink
-libegg instead of -libpy
Browse files Browse the repository at this point in the history
  • Loading branch information
Klaas Bosteels authored and Klaas Bosteels committed Sep 23, 2008
1 parent b881142 commit dc7269e
Showing 1 changed file with 4 additions and 4 deletions.
8 changes: 4 additions & 4 deletions dumbo.py
Expand Up @@ -139,22 +139,22 @@ def stream(prog,opts):
else: streamonhadoop(prog,opts,addedopts["hadoop"][0])

def streamlocally(prog,opts):
addedopts = delopts(opts,["input","output","mapper","reducer","libpy",
addedopts = delopts(opts,["input","output","mapper","reducer","libegg",
"delinputs"])
mapper,reducer = addedopts["mapper"][0],addedopts["reducer"][0]
if (not addedopts["input"]) or (not addedopts["output"]):
print >>sys.stderr,"ERROR: input or output not specified"
sys.exit(1)
input,output = addedopts["input"][0],addedopts["output"][0]
pythonenv = envdef("PYTHONPATH",addedopts["libpy"],opts)
pythonenv = envdef("PYTHONPATH",addedopts["libegg"],opts)
retval = execute("%s %s < '%s' | LC_ALL=C sort | %s %s > '%s'" % \
(pythonenv,mapper,input,pythonenv,reducer,output))
if addedopts["delinputs"] and addedopts["delinputs"][0] == "yes":
for file in addedopts["input"]: execute("rm " + file)
sys.exit(retval)

def streamonhadoop(prog,opts,hadoop):
addedopts = delopts(opts,["name","delinputs","libpy","libjar","inputformat",
addedopts = delopts(opts,["name","delinputs","libegg","libjar","inputformat",
"nummaptasks","numreducetasks"])
opts.append(("file",prog))
opts.append(("file",sys.argv[0]))
Expand Down Expand Up @@ -182,7 +182,7 @@ def streamonhadoop(prog,opts,hadoop):
inputformat_shortcuts[inputformat.lower()]
addedopts["libjar"].append(dumbojar)
opts.append(("inputformat",inputformat))
pythonenv = envdef("PYTHONPATH",addedopts["libpy"],opts)
pythonenv = envdef("PYTHONPATH",addedopts["libegg"],opts)
hadoopenv = envdef("HADOOP_CLASSPATH",addedopts["libjar"],opts)
cmd = hadoop + "/bin/hadoop jar " + streamingjar
retval = execute(cmd,opts," ".join((pythonenv,hadoopenv)))
Expand Down

0 comments on commit dc7269e

Please sign in to comment.