Permalink
Browse files

add libdirs parameter to findjars function

  • Loading branch information...
1 parent 4cedc5f commit 144e41134ae7a4b1c607230dc247badb3ff42c53 Klaas Bosteels committed Aug 20, 2012
Showing with 24 additions and 15 deletions.
  1. +3 −3 dumbo/backends/streaming.py
  2. +21 −12 dumbo/util.py
@@ -63,8 +63,7 @@ def run(self):
opts.remove(*keys)
hadoop = findhadoop(addedopts['hadoop'][0])
- hadooplib = findhadoop(addedopts['hadooplib'][0]) if 'hadooplib' in addedopts else hadoop
- streamingjar = findjar(hadooplib, 'streaming')
+ streamingjar = findjar(hadoop, 'streaming', addedopts['hadooplib'])
if not streamingjar:
print >> sys.stderr, 'ERROR: Streaming jar not found'
return 1
@@ -211,7 +210,8 @@ def __init__(self, hadoop):
self.hdfs = hadoop + '/bin/hadoop fs'
def cat(self, path, opts):
- streamingjar = findjar(self.hadoop, 'streaming')
+ streamingjar = findjar(self.hadoop, 'streaming',
+ opts['hadooplib'] if 'hadooplib' in opts else None)
if not streamingjar:
print >> sys.stderr, 'ERROR: Streaming jar not found'
return 1
View
@@ -272,21 +272,30 @@ def findhadoop(optval):
return hadoop
-def findjar(hadoop, name):
+def findjar(hadoop, name, libdirs=None):
"""Tries to find a JAR file based on given
hadoop home directory and component base name (e.g 'streaming')"""
- jardir_candidates = filter(os.path.exists, [
- os.path.join(hadoop, 'mapred', 'build', 'contrib', name),
- os.path.join(hadoop, 'build', 'contrib', name),
- os.path.join(hadoop, 'mapred', 'contrib', name, 'lib'),
- os.path.join(hadoop, 'contrib', name, 'lib'),
- os.path.join(hadoop, 'mapred', 'contrib', name),
- os.path.join(hadoop, 'contrib', name),
- os.path.join(hadoop, 'mapred', 'contrib'),
- os.path.join(hadoop, 'contrib'),
- hadoop
- ])
+ searchdirs = [hadoop]
+ if libdirs:
+ for libdir in libdirs:
+ if os.path.exists(libdir):
+ searchdirs.append(libdir)
+
+ jardir_candidates = []
+ for searchdir in searchdirs:
+ jardir_candidates += filter(os.path.exists, [
+ os.path.join(searchdir, 'mapred', 'build', 'contrib', name),
+ os.path.join(searchdir, 'build', 'contrib', name),
+ os.path.join(searchdir, 'mapred', 'contrib', name, 'lib'),
+ os.path.join(searchdir, 'contrib', name, 'lib'),
+ os.path.join(searchdir, 'mapred', 'contrib', name),
+ os.path.join(searchdir, 'contrib', name),
+ os.path.join(searchdir, 'mapred', 'contrib'),
+ os.path.join(searchdir, 'contrib'),
+ searchdir
+ ])
+
regex = re.compile(r'hadoop.*%s.*\.jar' % name)
for jardir in jardir_candidates:

0 comments on commit 144e411

Please sign in to comment.