Skip to content
Fetching contributors…
Cannot retrieve contributors at this time
192 lines (163 sloc) 5.4 KB
import sys
import os
import re
import subprocess
def sorted(iterable, piecesize=None, key=None, reverse=False):
if not piecesize:
values = list(iterable)
values.sort(key=key, reverse=reverse)
for value in values:
yield value
else: # piecewise sorted
sequence = iter(iterable)
while True:
values = list(sequence.next() for i in xrange(piecesize))
values.sort(key=key, reverse=reverse)
for value in values:
yield value
if len(values) < piecesize:
break
def dumpcode(outputs):
for output in outputs:
yield map(repr, output)
def loadcode(inputs):
for input in inputs:
try:
yield map(eval, input.split('\t', 1))
except (ValueError, TypeError):
print >> sys.stderr, 'WARNING: skipping bad input (%s)' % input
if os.environ.has_key('dumbo_debug'):
raise
incrcounter('Dumbo', 'Bad inputs', 1)
def dumptext(outputs):
newoutput = []
for output in outputs:
for item in output:
if not hasattr(item, '__iter__'):
newoutput.append(str(item))
else:
newoutput.append('\t'.join(map(str, item)))
yield newoutput
del newoutput[:]
def loadtext(inputs):
offset = 0
for input in inputs:
yield (offset, input)
offset += len(input)
def parseargs(args):
(opts, key, values) = ([], None, [])
for arg in args:
if arg[0] == '-' and len(arg) > 1:
if key:
opts.append((key, ' '.join(values)))
(key, values) = (arg[1:], [])
else:
values.append(arg)
if key:
opts.append((key, ' '.join(values)))
return opts
def getopts(opts, keys, delete=True):
askedopts = dict((key, []) for key in keys)
(key, delindexes) = (None, [])
for (index, (key, value)) in enumerate(opts):
key = key.lower()
if askedopts.has_key(key):
askedopts[key].append(value)
delindexes.append(index)
if delete:
for delindex in reversed(delindexes):
del opts[delindex]
return askedopts
def getopt(opts, key, delete=True):
return getopts(opts, [key], delete)[key]
def configopts(section, prog=None, opts=[]):
from ConfigParser import SafeConfigParser, NoSectionError
if prog:
defaults = {'prog': prog.split('/')[-1].split('.py', 1)[0]}
else:
defaults = {}
try:
defaults.update([('user', os.environ['USER']), ('pwd',
os.environ['PWD'])])
except KeyError:
pass
for (key, value) in opts:
defaults[key.lower()] = value
parser = SafeConfigParser(defaults)
parser.read(['/etc/dumbo.conf', os.environ['HOME'] + '/.dumborc'])
(results, excludes) = ([], set(defaults.iterkeys()))
try:
for (key, value) in parser.items(section):
if not key.lower() in excludes:
results.append((key.split('_', 1)[0], value))
except NoSectionError:
pass
return results
def execute(cmd,
opts=[],
precmd='',
printcmd=True,
stdout=sys.stdout,
stderr=sys.stderr):
if precmd:
cmd = ' '.join((precmd, cmd))
args = ' '.join("-%s '%s'" % (key, value) for (key, value) in opts)
if args:
cmd = ' '.join((cmd, args))
if printcmd:
print >> stderr, 'EXEC:', cmd
return system(cmd, stdout, stderr)
def system(cmd, stdout=sys.stdout, stderr=sys.stderr):
if sys.version[:3] == '2.4':
return os.system(cmd) / 256
proc = subprocess.Popen(cmd, shell=True, stdout=stdout,
stderr=stderr)
return os.waitpid(proc.pid, 0)[1] / 256
def findhadoop(optval):
(hadoop, hadoop_shortcuts) = (optval, dict(configopts('hadoops')))
if hadoop_shortcuts.has_key(hadoop.lower()):
hadoop = hadoop_shortcuts[hadoop.lower()]
if not os.path.exists(hadoop):
print >> sys.stderr, 'ERROR: directory %s does not exist' % hadoop
sys.exit(1)
return hadoop
def findjar(hadoop, name):
jardir = hadoop + '/build/contrib/' + name
if not os.path.exists(jardir):
jardir = hadoop + '/contrib/' + name + '/lib'
if not os.path.exists(jardir):
jardir = hadoop + '/contrib/' + name
if not os.path.exists(jardir):
jardir = hadoop + '/contrib'
regex = re.compile('hadoop.*' + name + "\.jar")
try:
return jardir + '/' + filter(regex.match, os.listdir(jardir))[-1]
except:
return None
def envdef(varname,
files,
optname=None,
opts=None,
commasep=False,
shortcuts={},
quote=True,
trim=False):
optvals = []
for file in files:
if shortcuts.has_key(file.lower()):
file = shortcuts[file.lower()]
optvals.append(file)
if not trim:
path = ':'.join(optvals)
else:
path = ':'.join(optval.split('/')[-1] for optval in optvals)
if optname and optvals:
if not commasep:
for optval in optvals:
opts.append((optname, optval))
else:
opts.append((optname, ','.join(optvals)))
if not quote:
return '%s=%s' % (varname, path)
else:
return '%s="%s"' % (varname, ':'.join((path, '$' + varname)))
Something went wrong with that request. Please try again.