Permalink
Browse files

bugfixes + version bump

  • Loading branch information...
1 parent e465a2f commit f09add8b49ccdb85dff3479e45070120070a7ab4 Klaas Bosteels committed Dec 8, 2008
Showing with 12 additions and 10 deletions.
  1. +9 −7 src/python/dumbo.py
  2. +1 −1 src/python/setup.py
  3. +2 −2 src/test/python/testcoding.py
View
@@ -46,8 +46,8 @@ def __init__(self,prog,opts):
self.prog,self.opts = prog,opts
self.opts += configopts("common",prog,self.opts)
def run(self):
- addedopts = getopts(self.opts,["fake","debug","python",
- "iteration","hadoop","starter"])
+ addedopts = getopts(self.opts,["fake","debug","python","iteration",
+ "itercount","hadoop","starter","name"])
if addedopts["fake"] and addedopts["fake"][0] == "yes":
def dummysystem(*args,**kwargs): return 0
global system
@@ -59,6 +59,11 @@ def dummysystem(*args,**kwargs): return 0
self.opts.append(("python",python))
if not addedopts["iteration"]: iter = 0
else: iter = int(addedopts["iteration"][0])
+ if not addedopts["itercount"]: itercnt = 0
+ else: itercnt = int(addedopts["itercount"][0])
+ if addedopts["name"]: name = addedopts["name"][0]
+ else: name = sys.argv[0].split("/")[-1]
+ self.opts.append(("name","%s (%s/%s)" % (name,iter+1,itercnt)))
if not addedopts["hadoop"]: progincmd = self.prog
else:
self.opts.append(("hadoop",addedopts["hadoop"][0]))
@@ -261,16 +266,13 @@ def run(mapper,reducer=None,combiner=None,
else:
opts = parseargs(sys.argv[1:])
newopts["iteration"] = str(iter)
+ newopts["itercount"] = str(itercnt)
if not reducer: newopts["numreducetasks"] = "0"
key,delindexes = None,[]
for index,(key,value) in enumerate(opts):
if newopts.has_key(key): delindexes.append(index)
for delindex in reversed(delindexes): del opts[delindex]
opts += newopts.iteritems()
- nameopt = getopts(opts,["name"])["name"]
- if nameopt: name = nameopt[0]
- else: name = sys.argv[0].split("/")[-1]
- opts.append(("name","%s (%s/%s)" % (name,iter+1,itercnt)))
hadoopopt = getopts(opts,["hadoop"],delete=False)["hadoop"]
if hadoopopt: retval = StreamingIteration(sys.argv[0],opts).run()
else: retval = UnixIteration(sys.argv[0],opts).run()
@@ -310,7 +312,7 @@ def dumpcode(outputs):
def loadcode(inputs):
for input in inputs:
try: yield map(eval,input.split("\t",1))
- except:
+ except (ValueError,TypeError):
if os.environ.has_key("debug"): raise
print >>sys.stderr,"WARNING: skipping bad input (%s)" % input
incrcounter("Dumbo","Bad inputs",1)
View
@@ -1,7 +1,7 @@
from distutils.core import setup
setup(name='dumbo',
- version='0.20.8',
+ version='0.20.9',
py_modules=['dumbo'],
author='Klaas Bosteels',
author_email='klaas@last.fm',
@@ -3,8 +3,8 @@
class TestCoding(unittest.TestCase):
def dotest(self,data):
- dumped = dumbo.dumpcode([[data]]).next()[0]
- self.assertEqual(dumbo.loadcode([dumped]).next()[0],data)
+ dumped = "\t".join(dumbo.dumpcode([("dummy",data)]).next())
+ self.assertEqual(dumbo.loadcode([dumped]).next()[1],data)
def testtuple(self):
self.dotest(tuple())

0 comments on commit f09add8

Please sign in to comment.